You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by db...@apache.org on 2023/03/07 13:49:06 UTC
[impala] 03/03: IMPALA-9551: Allow mixed complex types in select list
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2d47306987774d5fd134c798183b97dd81d26340
Author: Daniel Becker <da...@cloudera.com>
AuthorDate: Fri Oct 21 11:03:30 2022 +0200
IMPALA-9551: Allow mixed complex types in select list
Currently collections and structs are supported in the select list, also
when they are nested (structs in structs and collections in
collections), but mixing different kinds of complex types, i.e. having
structs in collections or vice versa, is not supported.
This patch adds support for mixed complex types in the select list.
Limitation: zipping unnest is not supported for mixed complex types, for
example the following query:
use functional_parquet;
select unnest(struct_contains_nested_arr.arr) from
collection_struct_mix;
Testing:
- Created a new test table, 'collection_struct_mix', that contains
mixed complex types.
- Added tests in mixed-collections-and-structs.test that test having
mixed complex types in the select list. These tests are called from
test_nested_types.py::TestMixedCollectionsAndStructsInSelectList.
- Ran existing tests that test collections and structs in the select
list; test queries that expected a failure in case of mixed complex
types have been moved to mixed-collections-and-structs.test and now
expect success.
Change-Id: I476d98884b5fd192dfcd4feeec7947526aebe993
Reviewed-on: http://gerrit.cloudera.org:8080/19322
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exec/unnest-node.cc | 9 +-
be/src/exprs/slot-ref.h | 1 +
be/src/runtime/complex-value-writer.h | 6 +-
be/src/runtime/complex-value-writer.inline.h | 74 ++-
be/src/runtime/descriptors.cc | 12 +-
be/src/runtime/raw-value.cc | 10 +-
be/src/service/hs2-util.cc | 23 +-
be/src/service/query-result-set.cc | 7 +-
.../java/org/apache/impala/analysis/Analyzer.java | 201 +++++---
.../apache/impala/analysis/CollectionTableRef.java | 8 +
.../org/apache/impala/analysis/InlineViewRef.java | 229 +++++----
.../org/apache/impala/analysis/SlotDescriptor.java | 26 +-
.../java/org/apache/impala/analysis/SlotRef.java | 4 -
.../apache/impala/analysis/TupleDescriptor.java | 2 +-
.../org/apache/impala/analysis/UnnestExpr.java | 41 +-
.../apache/impala/planner/SingleNodePlanner.java | 7 +-
.../apache/impala/analysis/AnalyzeStmtsTest.java | 24 +-
.../functional/functional_schema_template.sql | 131 ++++-
.../datasets/functional/schema_constraints.csv | 6 +
.../queries/QueryTest/map_null_keys.test | 14 +-
.../QueryTest/mixed-collections-and-structs.test | 551 +++++++++++++++++++++
.../ranger_column_masking_complex_types.test | 2 +-
.../queries/QueryTest/struct-in-select-list.test | 22 -
tests/query_test/test_nested_types.py | 32 ++
24 files changed, 1186 insertions(+), 256 deletions(-)
diff --git a/be/src/exec/unnest-node.cc b/be/src/exec/unnest-node.cc
index 36f084d72..f48596cab 100644
--- a/be/src/exec/unnest-node.cc
+++ b/be/src/exec/unnest-node.cc
@@ -62,7 +62,14 @@ Status UnnestPlanNode::InitCollExprs(FragmentState* state) {
SlotDescriptor* slot_desc = state->desc_tbl().GetSlotDescriptor(slot_ref->slot_id());
DCHECK(slot_desc != nullptr);
coll_slot_descs_.push_back(slot_desc);
- coll_tuple_idxs_.push_back(row_desc.GetTupleIdx(slot_desc->parent()->id()));
+
+ // If the collection is in a struct we don't use the itemTupleDesc of the struct but
+ // the tuple in which the top level struct is placed.
+ const TupleDescriptor* parent_tuple = slot_desc->parent();
+ const TupleDescriptor* master_tuple = parent_tuple->getMasterTuple();
+ const TupleDescriptor* top_level_tuple = master_tuple == nullptr ?
+ parent_tuple : master_tuple;
+ coll_tuple_idxs_.push_back(row_desc.GetTupleIdx(top_level_tuple->id()));
}
return Status::OK();
}
diff --git a/be/src/exprs/slot-ref.h b/be/src/exprs/slot-ref.h
index c0b1bf31a..7ebc1251d 100644
--- a/be/src/exprs/slot-ref.h
+++ b/be/src/exprs/slot-ref.h
@@ -63,6 +63,7 @@ class SlotRef : public ScalarExpr {
const SlotId& slot_id() const { return slot_id_; }
static const char* LLVM_CLASS_NAME;
NullIndicatorOffset GetNullIndicatorOffset() const { return null_indicator_offset_; }
+ const SlotDescriptor* GetSlotDescriptor() const { return slot_desc_; }
int GetSlotOffset() const { return slot_offset_; }
virtual const TupleDescriptor* GetCollectionTupleDesc() const override;
diff --git a/be/src/runtime/complex-value-writer.h b/be/src/runtime/complex-value-writer.h
index e1c7d05fc..5a85bbeb7 100644
--- a/be/src/runtime/complex-value-writer.h
+++ b/be/src/runtime/complex-value-writer.h
@@ -44,15 +44,17 @@ class ComplexValueWriter {
void CollectionValueToJSON(const CollectionValue& collection_value,
PrimitiveType collection_type, const TupleDescriptor* item_tuple_desc);
- // Gets a non-null StructVal and writes it in JSON format. Uses 'column_type' to figure
+ // Gets a non-null StructVal and writes it in JSON format. Uses 'slot_desc' to figure
// out field names and types. This function can call itself recursively in case of
// nested structs.
void StructValToJSON(const impala_udf::StructVal& struct_val,
- const ColumnType& column_type);
+ const SlotDescriptor& slot_desc);
private:
void PrimitiveValueToJSON(void* value, const ColumnType& type, bool map_key);
void WriteNull(bool map_key);
+ void StructInCollectionToJSON(Tuple* item_tuple,
+ const SlotDescriptor& struct_slot_desc);
void CollectionElementToJSON(Tuple* item_tuple, const SlotDescriptor& slot_desc,
bool map_key);
void ArrayValueToJSON(const CollectionValue& array_value,
diff --git a/be/src/runtime/complex-value-writer.inline.h b/be/src/runtime/complex-value-writer.inline.h
index e73a17b60..c53325e1d 100644
--- a/be/src/runtime/complex-value-writer.inline.h
+++ b/be/src/runtime/complex-value-writer.inline.h
@@ -81,6 +81,51 @@ void ComplexValueWriter<JsonStream>::WriteNull(bool map_key) {
}
}
+// Structs in collections are not converted to StructVal but are left as they are in the
+// tuple.
+template <class JsonStream>
+void ComplexValueWriter<JsonStream>::StructInCollectionToJSON(Tuple* item_tuple,
+ const SlotDescriptor& struct_slot_desc) {
+ const TupleDescriptor* children_tuple_desc =
+ struct_slot_desc.children_tuple_descriptor();
+ DCHECK(children_tuple_desc != nullptr);
+
+ const ColumnType& struct_type = struct_slot_desc.type();
+ const std::vector<SlotDescriptor*>& child_slots = children_tuple_desc->slots();
+
+ DCHECK(struct_type.type == TYPE_STRUCT);
+ DCHECK_EQ(child_slots.size(), struct_type.children.size());
+
+ writer_->StartObject();
+ for (int i = 0; i < child_slots.size(); ++i) {
+ writer_->String(struct_type.field_names[i].c_str());
+
+ const SlotDescriptor& child_slot_desc = *child_slots[i];
+ bool element_is_null = item_tuple->IsNull(child_slot_desc.null_indicator_offset());
+ if (element_is_null) {
+ WriteNull(writer_);
+ continue;
+ }
+
+ const ColumnType& child_type = child_slot_desc.type();
+ void* child = item_tuple->GetSlot(child_slot_desc.tuple_offset());
+ if (child_type.IsStructType()) {
+ StructInCollectionToJSON(item_tuple, child_slot_desc);
+ } else if (child_type.IsCollectionType()) {
+ const CollectionValue* nested_collection_val =
+ reinterpret_cast<CollectionValue*>(child);
+ const TupleDescriptor* child_item_tuple_desc =
+ child_slot_desc.children_tuple_descriptor();
+ DCHECK(child_item_tuple_desc != nullptr);
+ CollectionValueToJSON(*nested_collection_val, child_type.type,
+ child_item_tuple_desc);
+ } else {
+ PrimitiveValueToJSON(child, child_type, false);
+ }
+ }
+ writer_->EndObject();
+}
+
template <class JsonStream>
void ComplexValueWriter<JsonStream>::CollectionElementToJSON(Tuple* item_tuple,
const SlotDescriptor& slot_desc, bool map_key) {
@@ -91,7 +136,8 @@ void ComplexValueWriter<JsonStream>::CollectionElementToJSON(Tuple* item_tuple,
if (element_is_null) {
WriteNull(map_key);
} else if (element_type.IsStructType()) {
- DCHECK(false) << "Structs in collections are not supported yet.";
+ DCHECK(!map_key) << "Structs cannot be map keys.";
+ StructInCollectionToJSON(item_tuple, slot_desc);
} else if (element_type.IsCollectionType()) {
const CollectionValue* nested_collection_val =
reinterpret_cast<CollectionValue*>(element);
@@ -179,18 +225,32 @@ void ComplexValueWriter<JsonStream>::CollectionValueToJSON(
template <class JsonStream>
void ComplexValueWriter<JsonStream>::StructValToJSON(const StructVal& struct_val,
- const ColumnType& column_type) {
- DCHECK(column_type.type == TYPE_STRUCT);
- DCHECK_EQ(struct_val.num_children, column_type.children.size());
+ const SlotDescriptor& slot_desc) {
+ const ColumnType& struct_type = slot_desc.type();
+ DCHECK(struct_type.type == TYPE_STRUCT);
+ DCHECK_EQ(struct_val.num_children, struct_type.children.size());
+
+ const TupleDescriptor* children_item_tuple_desc = slot_desc.children_tuple_descriptor();
+ DCHECK(children_item_tuple_desc != nullptr);
+ const std::vector<SlotDescriptor*>& child_slot_descs =
+ children_item_tuple_desc->slots();
+ DCHECK_EQ(struct_val.num_children, child_slot_descs.size());
+
writer_->StartObject();
for (int i = 0; i < struct_val.num_children; ++i) {
- writer_->String(column_type.field_names[i].c_str());
+ writer_->String(struct_type.field_names[i].c_str());
void* child = (void*)(struct_val.ptr[i]);
- const ColumnType& child_type = column_type.children[i];
+ const SlotDescriptor& child_slot_desc = *child_slot_descs[i];
+ const ColumnType& child_type = struct_type.children[i];
+ DCHECK_EQ(child_type, child_slot_desc.type());
if (child == nullptr) {
WriteNull(false);
} else if (child_type.IsStructType()) {
- StructValToJSON(*((StructVal*)child), child_type);
+ StructValToJSON(*((StructVal*)child), child_slot_desc);
+ } else if (child_type.IsCollectionType()) {
+ CollectionValue* collection_child = reinterpret_cast<CollectionValue*>(child);
+ ComplexValueWriter::CollectionValueToJSON(*collection_child, child_type.type,
+ child_slot_desc.children_tuple_descriptor());
} else {
PrimitiveValueToJSON(child, child_type, false);
}
diff --git a/be/src/runtime/descriptors.cc b/be/src/runtime/descriptors.cc
index 2b3ffc92f..47234e168 100644
--- a/be/src/runtime/descriptors.cc
+++ b/be/src/runtime/descriptors.cc
@@ -352,17 +352,17 @@ TupleDescriptor::TupleDescriptor(const TTupleDescriptor& tdesc)
void TupleDescriptor::AddSlot(SlotDescriptor* slot) {
slots_.push_back(slot);
+ // If this is a tuple for struct children then we populate the 'string_slots_' field (in
+ // case of a var len string type) or the 'collection_slots_' field (in case of a
+ // collection type) of the topmost tuple and not this one.
+ TupleDescriptor* const target_tuple = isTupleOfStructSlot() ? master_tuple_ : this;
if (slot->type().IsVarLenStringType()) {
- TupleDescriptor* target_tuple = this;
- // If this is a tuple for struct children then we populate the 'string_slots_' of
- // the topmost tuple and not this one.
- if (isTupleOfStructSlot()) target_tuple = master_tuple_;
target_tuple->string_slots_.push_back(slot);
target_tuple->has_varlen_slots_ = true;
}
if (slot->type().IsCollectionType()) {
- collection_slots_.push_back(slot);
- has_varlen_slots_ = true;
+ target_tuple->collection_slots_.push_back(slot);
+ target_tuple->has_varlen_slots_ = true;
}
}
diff --git a/be/src/runtime/raw-value.cc b/be/src/runtime/raw-value.cc
index 32ca90e04..be8058db0 100644
--- a/be/src/runtime/raw-value.cc
+++ b/be/src/runtime/raw-value.cc
@@ -301,7 +301,7 @@ void RawValue::PrintValue(
case TYPE_BOOLEAN: {
bool val = *reinterpret_cast<const bool*>(value);
*stream << (val ? "true" : "false");
- return;
+ break;
}
case TYPE_TINYINT:
// Extra casting for chars since they should not be interpreted as ASCII.
@@ -410,12 +410,4 @@ template void RawValue::WritePrimitive<true>(const void* value, Tuple* tuple,
template void RawValue::WritePrimitive<false>(const void* value, Tuple* tuple,
const SlotDescriptor* slot_desc, MemPool* pool,
std::vector<StringValue*>* string_values);
-
-bool PrintNestedValueIfNull(const SlotDescriptor& slot_desc, Tuple* item,
- stringstream* stream) {
- bool is_null = item->IsNull(slot_desc.null_indicator_offset());
- if (is_null) *stream << RawValue::NullLiteral(false);
- return is_null;
-}
-
}
diff --git a/be/src/service/hs2-util.cc b/be/src/service/hs2-util.cc
index b3255d545..20279699c 100644
--- a/be/src/service/hs2-util.cc
+++ b/be/src/service/hs2-util.cc
@@ -26,6 +26,7 @@
#include "common/logging.h"
#include "exprs/scalar-expr.h"
#include "exprs/scalar-expr-evaluator.h"
+#include "exprs/slot-ref.h"
#include "gen-cpp/TCLIService_constants.h"
#include "runtime/date-value.h"
#include "runtime/complex-value-writer.inline.h"
@@ -371,7 +372,8 @@ static void DecimalExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
static void StructExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
const TColumnType& type, RowBatch* batch, int start_idx, int num_rows,
- uint32_t output_row_idx, apache::hive::service::cli::thrift::TColumn* column) {
+ uint32_t output_row_idx, bool stringify_map_keys,
+ apache::hive::service::cli::thrift::TColumn* column) {
DCHECK(type.types.size() > 1);
ReserveSpace(num_rows, output_row_idx, &column->stringVal);
// The buffer used by rapidjson::Writer. We reuse it to eliminate allocations.
@@ -381,16 +383,19 @@ static void StructExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
if (struct_val.is_null) {
column->stringVal.values.emplace_back();
} else {
- int idx = 0;
- ColumnType column_type(type.types, &idx);
+ const impala::ScalarExpr& scalar_expr = expr_eval->root();
+ // Currently scalar_expr can be only a slot ref as no functions return arrays.
+ DCHECK(scalar_expr.IsSlotRef());
+ const SlotDescriptor* slot_desc =
+ static_cast<const SlotRef&>(scalar_expr).GetSlotDescriptor();
+ DCHECK(slot_desc != nullptr);
buffer.Clear();
rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
- // TODO: Create a stringify_map_keys parameter and pass that when
- // "IMPALA-9551: Allow mixed complex types in select list" is done.
- ComplexValueWriter<rapidjson::StringBuffer> complex_value_writer(&writer, false);
- complex_value_writer.StructValToJSON(struct_val, column_type);
+ ComplexValueWriter<rapidjson::StringBuffer> complex_value_writer(&writer,
+ stringify_map_keys);
+ complex_value_writer.StructValToJSON(struct_val, *slot_desc);
column->stringVal.values.emplace_back(buffer.GetString());
}
@@ -451,8 +456,8 @@ void impala::ExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
// to inline the expression evaluation into the loop body.
switch (type.types[0].type) {
case TTypeNodeType::STRUCT:
- StructExprValuesToHS2TColumn(
- expr_eval, type, batch, start_idx, num_rows, output_row_idx, column);
+ StructExprValuesToHS2TColumn(expr_eval, type, batch, start_idx, num_rows,
+ output_row_idx, stringify_map_keys, column);
return;
case TTypeNodeType::ARRAY:
case TTypeNodeType::MAP:
diff --git a/be/src/service/query-result-set.cc b/be/src/service/query-result-set.cc
index e90efe7ea..dd7a2b2f0 100644
--- a/be/src/service/query-result-set.cc
+++ b/be/src/service/query-result-set.cc
@@ -252,9 +252,14 @@ void QueryResultSet::PrintComplexValue(ScalarExprEvaluator* expr_eval,
} else {
DCHECK(type.IsStructType());
const StructVal* struct_val = static_cast<const StructVal*>(value);
+ const SlotDescriptor* slot_desc =
+ static_cast<const SlotRef&>(scalar_expr).GetSlotDescriptor();
+ DCHECK(slot_desc != nullptr);
+ DCHECK_EQ(type, slot_desc->type());
+
ComplexValueWriter<rapidjson::BasicOStreamWrapper<stringstream>>
complex_value_writer(&writer, stringify_map_keys);
- complex_value_writer.StructValToJSON(*struct_val, type);
+ complex_value_writer.StructValToJSON(*struct_val, *slot_desc);
}
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 2acdea7a9..b5945eb76 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -17,10 +17,12 @@
package org.apache.impala.analysis;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
@@ -238,6 +240,48 @@ public class Analyzer {
// to be applied of the Iceberg V2 table.
private long totalRecordsNumV2_;
+ // The method 'registerSlotRef()' is called recursively for complex types (structs and
+ // collections). When creating new 'SlotDescriptor's it is important that they are
+ // inserted into the correct 'TupleDescriptor'. This stack is used to keep track of the
+ // current (most nested) 'TupleDescriptor' at each step.
+ //
+ // When a new top-level path is registered, the root descriptor of the path is pushed on
+ // the stack. When registering a complex type, we push its item tuple desc on the stack
+ // before analysing the children, and pop it afterwards. New 'SlotDescriptor's are
+ // always added to the 'TupleDescriptor' at the top of the stack.
+ //
+ // To ensure that every push operation has a corresponding pop operation, these should
+ // not be called manually. Instead, use 'TupleStackGuard' objects in try-with-resources
+ // blocks; see more in its documentation.
+ private Deque<TupleDescriptor> tupleStack_ = new ArrayDeque<>();
+
+ // When created, pushes the provided TupleDescriptor to the enclosing Analyzer object's
+ // 'tupleStack_' and pops it in the close() method. Implements the AutoCloseable
+ // interface so it can be used in try-with-resources blocks.
+ private class TupleStackGuard implements AutoCloseable {
+ private final TupleDescriptor tupleDesc_;
+ private boolean isClosed_ = false;
+ private final int stackLen_;
+
+ public TupleStackGuard(TupleDescriptor tupleDesc) {
+ Preconditions.checkNotNull(tupleStack_);
+ Preconditions.checkNotNull(tupleDesc);
+ tupleDesc_ = tupleDesc;
+ stackLen_ = tupleStack_.size();
+ tupleStack_.push(tupleDesc_);
+ }
+
+ @Override
+ public void close() {
+ if (!isClosed_) {
+ Preconditions.checkState(tupleStack_.peek() == tupleDesc_);
+ tupleStack_.pop();
+ Preconditions.checkState(tupleStack_.size() == stackLen_);
+ isClosed_ = true;
+ }
+ }
+ }
+
// Required Operation type: Read, write, any(read or write).
public enum OperationType {
READ,
@@ -1496,55 +1540,74 @@ public class Analyzer {
/**
* Returns an existing or new SlotDescriptor for the given path.
- * If duplicateIfCollections is true, then always returns
- * a new empty SlotDescriptor for paths with a collection-typed destination.
+ * If 'duplicateIfCollections' is true, then always returns a new empty SlotDescriptor
+ * for paths with a collection-typed destination.
*/
- public SlotDescriptor registerSlotRef(Path slotPath, boolean duplicateIfCollections)
- throws AnalysisException {
+ public SlotDescriptor registerSlotRef(Path slotPath,
+ boolean duplicateIfCollections) throws AnalysisException {
Preconditions.checkState(slotPath.isRootedAtTuple());
- if (slotPath.destType().isCollectionType() && duplicateIfCollections) {
- // Register a new slot descriptor for collection types. The BE currently
- // relies on this behavior for setting unnested collection slots to NULL.
- return createAndRegisterSlotDesc(slotPath, false);
- }
- // SlotRefs with scalar or struct types are registered against the slot's
- // fully-qualified lowercase path.
- List<String> key = slotPath.getFullyQualifiedRawPath();
- Preconditions.checkState(key.stream().allMatch(s -> s.equals(s.toLowerCase())),
- "Slot paths should be lower case: " + key);
- SlotDescriptor existingSlotDesc = slotPathMap_.get(key);
- if (existingSlotDesc != null) return existingSlotDesc;
+ // If 'tupleStack_' is empty then this is a top level call to this function (not a
+ // recursive call) and we push the root TupleDescriptor to 'tupleStack_'.
+ try (TupleStackGuard guard = tupleStack_.isEmpty()
+ ? new TupleStackGuard(slotPath.getRootDesc()) : null) {
+ if (slotPath.destType().isCollectionType() && duplicateIfCollections) {
+ // Register a new slot descriptor for collection types. The BE currently
+ // relies on this behavior for setting unnested collection slots to NULL.
+ return createAndRegisterRawSlotDesc(slotPath, false);
+ }
+ // SlotRefs with scalar or struct types are registered against the slot's
+ // fully-qualified lowercase path.
+ List<String> key = slotPath.getFullyQualifiedRawPath();
+ Preconditions.checkState(key.stream().allMatch(s -> s.equals(s.toLowerCase())),
+ "Slot paths should be lower case: " + key);
+ SlotDescriptor existingSlotDesc = slotPathMap_.get(key);
+ if (existingSlotDesc != null) return existingSlotDesc;
+
+ return createAndRegisterSlotDesc(slotPath);
+ }
+ }
+ private SlotDescriptor createAndRegisterSlotDesc(Path slotPath)
+ throws AnalysisException {
if (slotPath.destType().isCollectionType()) {
SlotDescriptor result = registerCollectionSlotRef(slotPath);
- result.setPath(slotPath);
- slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), result);
- registerColumnPrivReq(result);
+ registerSlotDesc(slotPath, result, true);
return result;
}
+ SlotDescriptor result = createAndRegisterRawSlotDesc(slotPath, true);
if (slotPath.destType().isStructType()) {
- // 'slotPath' refers to a struct that has no ancestor that is also needed in the
- // query. We also create the child slot descriptors.
- SlotDescriptor result = createAndRegisterSlotDesc(slotPath, true);
createStructTuplesAndSlotDescs(result);
- return result;
}
+ return result;
+ }
- SlotDescriptor result = createAndRegisterSlotDesc(slotPath, true);
+ /**
+ * Creates a new SlotDescriptor with path 'slotPath' in the tuple at the top of
+ * 'tupleStack_'. Does not create SlotDescriptors for children of complex types. If
+ * 'insertIntoSlotPath' is true, inserts the new SlotDescriptor into 'slotPathMap_'.
+ * Also registers a column-level privilege request for the new SlotDescriptor.
+ */
+ private SlotDescriptor createAndRegisterRawSlotDesc(Path slotPath,
+ boolean insertIntoSlotPathMap) {
+ final SlotDescriptor result = addSlotDescriptorAtCurrentLevel();
+ registerSlotDesc(slotPath, result, insertIntoSlotPathMap);
return result;
}
- private SlotDescriptor createAndRegisterSlotDesc(Path slotPath,
+ /**
+ * Sets 'slotPath' as the path of 'desc' and registers a column-level privilege request
+ * for it. If 'insertIntoSlotPath' is true, inserts the new SlotDescriptor into
+ * 'slotPathMap_'.
+ */
+ private void registerSlotDesc(Path slotPath, SlotDescriptor desc,
boolean insertIntoSlotPathMap) {
Preconditions.checkState(slotPath.isRootedAtTuple());
- final SlotDescriptor result = addSlotDescriptor(slotPath.getRootDesc());
- result.setPath(slotPath);
+ desc.setPath(slotPath);
if (insertIntoSlotPathMap) {
- slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), result);
+ slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), desc);
}
- registerColumnPrivReq(result);
- return result;
+ registerColumnPrivReq(desc);
}
public void createStructTuplesAndSlotDescs(SlotDescriptor desc)
@@ -1557,26 +1620,38 @@ public class Analyzer {
structTuple.setType(type);
structTuple.setParentSlotDesc(desc);
desc.setItemTupleDesc(structTuple);
- for (StructField structField : type.getFields()) {
- SlotDescriptor childDesc = getDescTbl().addSlotDescriptor(structTuple);
- globalState_.blockBySlot.put(childDesc.getId(), this);
- // 'slotPath' could be null e.g. when the query has an order by clause and
- // this is the sorting tuple.
- if (slotPath != null) {
- Path relPath = Path.createRelPath(slotPath, structField.getName());
- relPath.resolve();
- childDesc.setPath(relPath);
- registerColumnPrivReq(childDesc);
- slotPathMap_.putIfAbsent(relPath.getFullyQualifiedRawPath(), childDesc);
- }
- childDesc.setType(structField.getType());
- if (childDesc.getType().isStructType()) {
- createStructTuplesAndSlotDescs(childDesc);
+ try (TupleStackGuard guard = new TupleStackGuard(structTuple)) {
+ for (StructField structField : type.getFields()) {
+ // 'slotPath' could be null e.g. when the query has an order by clause and
+ // this is the sorting tuple.
+ // TODO: IMPALA-10939: When we enable collections in sorting tuples we need to
+ // revisit this. Currently collection SlotDescriptors cannot be created without a
+ // path.
+ if (slotPath == null) {
+ createStructTuplesAndSlotDescsWithoutPath(slotPath, structField);
+ } else {
+ Path childRelPath = Path.createRelPath(slotPath, structField.getName());
+ childRelPath.resolve();
+ SlotDescriptor childDesc = createAndRegisterSlotDesc(childRelPath);
+ }
}
}
}
+ private void createStructTuplesAndSlotDescsWithoutPath(Path slotPath,
+ StructField structField) throws AnalysisException {
+ Preconditions.checkState(slotPath == null);
+ Preconditions.checkState(!structField.getType().isCollectionType());
+
+ SlotDescriptor childDesc = addSlotDescriptorAtCurrentLevel();
+ childDesc.setType(structField.getType());
+
+ if (childDesc.getType().isStructType()) {
+ createStructTuplesAndSlotDescs(childDesc);
+ }
+ }
+
/**
* Registers a collection and its descendants.
* Creates a CollectionTableRef for all collections on the path.
@@ -1604,18 +1679,20 @@ public class Analyzer {
SlotDescriptor desc = ((SlotRef) collTblRef.getCollectionExpr()).getDesc();
desc.setIsMaterializedRecursively(true);
- if (slotPath.destType().isArrayType()) {
- // Resolve path
- List<String> rawPathToItem = Arrays.asList(Path.ARRAY_ITEM_FIELD_NAME);
- resolveAndRegisterDescendantPath(collTblRef, rawPathToItem);
- } else {
- Preconditions.checkState(slotPath.destType().isMapType());
+ try (TupleStackGuard guard = new TupleStackGuard(desc.getItemTupleDesc())) {
+ if (slotPath.destType().isArrayType()) {
+ // Resolve path
+ List<String> rawPathToItem = Arrays.asList(Path.ARRAY_ITEM_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToItem);
+ } else {
+ Preconditions.checkState(slotPath.destType().isMapType());
- List<String> rawPathToKey = Arrays.asList(Path.MAP_KEY_FIELD_NAME);
- resolveAndRegisterDescendantPath(collTblRef, rawPathToKey);
+ List<String> rawPathToKey = Arrays.asList(Path.MAP_KEY_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToKey);
- List<String> rawPathToValue = Arrays.asList(Path.MAP_VALUE_FIELD_NAME);
- resolveAndRegisterDescendantPath(collTblRef, rawPathToValue);
+ List<String> rawPathToValue = Arrays.asList(Path.MAP_VALUE_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToValue);
+ }
}
return desc;
@@ -1627,10 +1704,6 @@ public class Analyzer {
boolean isResolved = resolvedPath.resolve();
Preconditions.checkState(isResolved);
- if (resolvedPath.destType().isStructType()) {
- throw new AnalysisException(
- "STRUCT type inside collection types is not supported.");
- }
if (resolvedPath.destType().isBinary()) {
throw new AnalysisException(
"Binary type inside collection types is not supported (IMPALA-11491).");
@@ -1698,6 +1771,16 @@ public class Analyzer {
return result;
}
+ /**
+ * Creates a new slot descriptor and related state in globalState. The new slot
+ * descriptor will be created in the tuple at the top of 'tupleStack_'.
+ */
+ private SlotDescriptor addSlotDescriptorAtCurrentLevel() {
+ TupleDescriptor tupleDesc = tupleStack_.peek();
+ Preconditions.checkNotNull(tupleDesc);
+ return addSlotDescriptor(tupleDesc);
+ }
+
/**
* Adds a new slot descriptor in tupleDesc that is identical to srcSlotDesc
* except for the path and slot id.
diff --git a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
index 2656886d5..1199bd086 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
@@ -19,6 +19,7 @@ package org.apache.impala.analysis;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.FeView;
+import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
import com.google.common.base.Preconditions;
@@ -99,6 +100,13 @@ public class CollectionTableRef extends TableRef {
if (resolvedPath_.getRootDesc() != null) {
sourceView = resolvedPath_.getRootDesc().getSourceView();
}
+ if (zippingUnnestType_ == ZippingUnnestType.FROM_CLAUSE_ZIPPING_UNNEST) {
+ UnnestExpr.verifyNotInsideStruct(resolvedPath_);
+
+ Type type = resolvedPath_.getMatchedTypes().get(
+ resolvedPath_.getMatchedTypes().size() - 1);
+ UnnestExpr.verifyContainsNoStruct(type);
+ }
if (sourceView != null && zippingUnnestType_ ==
ZippingUnnestType.FROM_CLAUSE_ZIPPING_UNNEST) {
String implicitAlias = rawPath_.get(rawPath_.size() - 1).toLowerCase();
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index 38c9508b5..2a57e6ec8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -297,137 +297,152 @@ public class InlineViewRef extends TableRef {
SlotDescriptor slotDesc = analyzer.registerSlotRef(p, false);
slotDesc.setSourceExpr(colExpr);
slotDesc.setStats(ColumnStats.fromExpr(colExpr));
- SlotRef slotRef = new SlotRef(slotDesc);
- smap_.put(slotRef, colExpr);
- baseTblSmap_.put(slotRef, baseTableExpr);
+
+ putExprsIntoSmaps(analyzer, slotDesc, colExpr, baseTableExpr);
+ }
+
+ // Inserts elements into 'smap_' and 'baseTblSmap_'. The key will be a new slot ref
+ // created from 'slotDesc' in both smaps; the value will be 'colExpr' in 'smap_' and
+ // 'baseTableExpr' in 'baseTblSmap_'. If 'recurse' is true, also adds struct members and
+ // collection items.
+ private void putExprsIntoSmaps(Analyzer analyzer,
+ SlotDescriptor slotDesc, Expr colExpr, Expr baseTableExpr, boolean recurse) {
+ SlotRef key = new SlotRef(slotDesc);
+
+ smap_.put(key, colExpr);
+ baseTblSmap_.put(key, baseTableExpr);
+
if (createAuxPredicate(colExpr)) {
analyzer.createAuxEqPredicate(new SlotRef(slotDesc), colExpr.clone());
}
- if (colExpr.getType().isCollectionType()) {
- // Calling registerSlotRef() above created a new SlotDescriptor + TupleDescriptor
- // hierarchy for Array types. Walk through this hiararchy and add all slot refs
- // to smap_ and baseTblSmap_.
- // Source must be a SlotRef
- SlotRef srcSlotRef = (SlotRef) colExpr;
- SlotRef baseTableSlotRef = (SlotRef) baseTableExpr;
- SlotDescriptor srcSlotDesc = srcSlotRef.getDesc();
- SlotDescriptor baseTableSlotDesc = baseTableSlotRef.getDesc();
- TupleDescriptor itemTupleDesc = slotDesc.getItemTupleDesc();
- TupleDescriptor srcItemTupleDesc = srcSlotDesc.getItemTupleDesc();
- TupleDescriptor baseTableItemTupleDesc = baseTableSlotDesc.getItemTupleDesc();
- // We don't recurse deeper and only add the immediate item child to the
- // substitution map. This is enough both for collections in select list and in
- // from clause.
- if (itemTupleDesc != null) {
- Preconditions.checkState(srcItemTupleDesc != null);
- Preconditions.checkState(baseTableItemTupleDesc != null);
-
- final int num_slots = itemTupleDesc.getSlots().size();
- Preconditions.checkState(srcItemTupleDesc.getSlots().size() == num_slots);
- Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() == num_slots);
-
- // There is one slot for arrays and two for maps. When we add support for structs
- // in collections in the select list, there may be more slots.
- for (int i = 0; i < num_slots; i++) {
- SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(i);
- SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(i);
- SlotDescriptor baseTableItemSlotDesc = baseTableItemTupleDesc.getSlots().get(i);
- SlotRef itemSlotRef = new SlotRef(itemSlotDesc);
- SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc);
- SlotRef beseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc);
- smap_.put(itemSlotRef, srcItemSlotRef);
- baseTblSmap_.put(itemSlotRef, beseTableItemSlotRef);
- if (createAuxPredicate(colExpr)) {
- analyzer.createAuxEqPredicate(
- new SlotRef(itemSlotDesc), srcItemSlotRef.clone());
- }
- }
- }
- }
+ if (recurse) {
+ if (colExpr.getType().isCollectionType()) {
+ Preconditions.checkState(colExpr instanceof SlotRef);
+ Preconditions.checkState(baseTableExpr instanceof SlotRef);
- if (colExpr.getType().isStructType()) {
- Preconditions.checkState(colExpr instanceof SlotRef);
- Preconditions.checkState(baseTableExpr instanceof SlotRef);
- putStructMembersIntoSmap(smap_, slotDesc, (SlotRef) colExpr);
- putStructMembersIntoSmap(baseTblSmap_, slotDesc, (SlotRef) baseTableExpr);
- createAuxPredicatesForStructMembers(analyzer, slotDesc, (SlotRef) colExpr);
- }
- }
+ putCollectionItemsIntoSmaps(analyzer, slotDesc, (SlotRef) colExpr,
+ (SlotRef) baseTableExpr);
+ } else if (colExpr.getType().isStructType()) {
+ Preconditions.checkState(colExpr instanceof SlotRef);
+ Preconditions.checkState(baseTableExpr instanceof SlotRef);
- // Puts the fields of 'rhsStruct' into 'smap' as right hand side (mapped) values,
- // recursively. The keys (left hand side values) are constructed based on the
- // corresponding slot descriptors in the itemTupleDesc of 'lhsSlotDesc'.
- private void putStructMembersIntoSmap(ExprSubstitutionMap smap,
- SlotDescriptor lhsSlotDesc, SlotRef rhsStruct) {
- for (Pair<SlotDescriptor, SlotRef> pair :
- getStructSlotDescSlotRefPairs(lhsSlotDesc, rhsStruct)) {
- SlotDescriptor lhs = pair.first;
- SlotRef rhs = pair.second;
- smap.put(new SlotRef(lhs), rhs);
+ putStructMembersIntoSmaps(analyzer, slotDesc, (SlotRef) colExpr,
+ (SlotRef) baseTableExpr);
+ }
}
}
- private void createAuxPredicatesForStructMembers(Analyzer analyzer,
- SlotDescriptor structSlotDesc, SlotRef structExpr) {
- for (Pair<SlotDescriptor, SlotRef> pair :
- getStructSlotDescSlotRefPairs(structSlotDesc, structExpr)) {
- SlotDescriptor structMemberSlotDesc = pair.first;
- SlotRef structMemberExpr = pair.second;
+ private void putExprsIntoSmaps(Analyzer analyzer,
+ SlotDescriptor slotDesc, Expr colExpr, Expr baseTableExpr) {
+ putExprsIntoSmaps(analyzer, slotDesc, colExpr, baseTableExpr, true);
+ }
- if (createAuxPredicate(structMemberExpr)) {
- analyzer.createAuxEqPredicate(
- new SlotRef(structMemberSlotDesc), structMemberExpr.clone());
+ // Add slot refs for collection items to smap_ and baseTblSmap_.
+ private void putCollectionItemsIntoSmaps(Analyzer analyzer, SlotDescriptor slotDesc,
+ SlotRef colExpr, SlotRef baseTableExpr) {
+ // Source must be a SlotRef
+ SlotDescriptor srcSlotDesc = colExpr.getDesc();
+ SlotDescriptor baseTableSlotDesc = baseTableExpr.getDesc();
+
+ TupleDescriptor itemTupleDesc = slotDesc.getItemTupleDesc();
+ TupleDescriptor srcItemTupleDesc = srcSlotDesc.getItemTupleDesc();
+ TupleDescriptor baseTableItemTupleDesc = baseTableSlotDesc.getItemTupleDesc();
+ if (itemTupleDesc != null) {
+ Preconditions.checkState(srcItemTupleDesc != null);
+ Preconditions.checkState(baseTableItemTupleDesc != null);
+
+ final int num_slots = itemTupleDesc.getSlots().size();
+ // There is one slot for arrays and two for maps.
+ Preconditions.checkState(num_slots == 1 || num_slots == 2);
+ Preconditions.checkState(srcItemTupleDesc.getSlots().size() == num_slots);
+ Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() == num_slots);
+
+ for (int i = 0; i < num_slots; i++) {
+ SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(i);
+ SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(i);
+ SlotDescriptor baseTableItemSlotDesc = baseTableItemTupleDesc.getSlots().get(i);
+ SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc);
+ SlotRef baseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc);
+
+ Preconditions.checkState(itemSlotDesc.getType().equals(srcItemSlotRef.getType()));
+ Preconditions.checkState(itemSlotDesc.getType().equals(
+ baseTableItemSlotRef.getType()));
+
+ // We don't recurse deeper and only add the immediate item child to the
+ // substitution map. This is enough both for collections in select list and in
+ // from clause.
+ putExprsIntoSmaps(analyzer, itemSlotDesc, srcItemSlotRef, baseTableItemSlotRef,
+ false);
}
}
}
- /**
- * Given a slot desc and a SlotRef expression, both referring to the same struct,
- * returns a list of corresponding SlotDescriptor/SlotRef pairs of the struct members,
- * recursively.
- */
- private List<Pair<SlotDescriptor, SlotRef>> getStructSlotDescSlotRefPairs(
- SlotDescriptor structSlotDesc, SlotRef structExpr) {
- Preconditions.checkState(structSlotDesc.getType().isStructType());
- Preconditions.checkState(structExpr.getType().isStructType());
- Preconditions.checkState(structSlotDesc.getType().equals(structExpr.getType()));
+ // Put struct members into 'smap_' and 'baseTblSmap_'. 'slotDesc', 'colExpr' and
+ // 'baseTableExpr' should all belong to the same struct. The struct tree is traversed;
+ // keys in both maps will be slot refs created from the elements of the tree rooted at
+ // 'slotDesc'. The values in 'smap_' will be the expressions in the tree of 'colExpr'
+ // and the values in 'baseTblSmap_' will be the expressions in the tree of
+ // 'baseTableExpr'
+ private void putStructMembersIntoSmaps(Analyzer analyzer, SlotDescriptor slotDesc,
+ SlotRef colExpr, SlotRef baseTableExpr) {
+ Preconditions.checkState(slotDesc.getType().isStructType());
+ Preconditions.checkState(colExpr.getType().isStructType());
+ Preconditions.checkState(baseTableExpr.getType().isStructType());
- List<Pair<SlotDescriptor, SlotRef>> result = new ArrayList<>();
+ Preconditions.checkState(slotDesc.getType().equals(colExpr.getType()));
+ Preconditions.checkState(slotDesc.getType().equals(baseTableExpr.getType()));
- TupleDescriptor lhsItemTupleDesc = structSlotDesc.getItemTupleDesc();
- Preconditions.checkNotNull(lhsItemTupleDesc);
- List<SlotDescriptor> lhsChildSlotDescs = lhsItemTupleDesc.getSlots();
- Preconditions.checkState(
- lhsChildSlotDescs.size() == structExpr.getChildren().size());
- for (int i = 0; i < lhsChildSlotDescs.size(); i++) {
- SlotDescriptor lhsChildSlotDesc = lhsChildSlotDescs.get(i);
+ TupleDescriptor itemTupleDesc = slotDesc.getItemTupleDesc();
+ Preconditions.checkNotNull(itemTupleDesc);
- Expr rhsChild = structExpr.getChildren().get(i);
- Preconditions.checkState(rhsChild instanceof SlotRef);
- SlotRef rhsChildSlotRef = (SlotRef) rhsChild;
+ List<SlotDescriptor> childSlotDescs = itemTupleDesc.getSlots();
+ Preconditions.checkState(childSlotDescs.size() == colExpr.getChildren().size());
+ Preconditions.checkState(childSlotDescs.size() == baseTableExpr.getChildren().size());
- List<String> lhsRawPath = lhsChildSlotDesc.getPath().getRawPath();
- Path rhsPath = rhsChildSlotRef.getResolvedPath();
+ for (int i = 0; i < childSlotDescs.size(); i++) {
+ SlotDescriptor childSlotDesc = childSlotDescs.get(i);
- // The path can be null in the case of the sorting tuple.
- if (rhsPath != null) {
- List<String> rhsRawPath = rhsPath.getRawPath();
+ Expr childColExpr = colExpr.getChildren().get(i);
+ Preconditions.checkState(childColExpr instanceof SlotRef);
+ SlotRef childColExprSlotRef = (SlotRef) childColExpr;
- // Check that the children come in the same order on both lhs and rhs. If not, the
- // last part of the paths would be different.
- Preconditions.checkState(lhsRawPath.get(lhsRawPath.size() - 1)
- .equals(rhsRawPath.get(rhsRawPath.size() - 1)));
+ Expr childBaseTableExpr = baseTableExpr.getChildren().get(i);
+ Preconditions.checkState(childBaseTableExpr instanceof SlotRef);
+ SlotRef childBaseTableExprSlotRef = (SlotRef) childBaseTableExpr;
- result.add(new Pair(lhsChildSlotDesc, rhsChildSlotRef));
+ Path childColExprPath = childColExprSlotRef.getResolvedPath();
+ Path childBaseTableExprPath = childBaseTableExprSlotRef.getResolvedPath();
- if (rhsChildSlotRef.getType().isStructType()) {
- result.addAll(
- getStructSlotDescSlotRefPairs(lhsChildSlotDesc, rhsChildSlotRef));
- }
+ // The path can be null in the case of the sorting tuple.
+ if (childColExprPath != null) {
+ verifySameChild(childSlotDesc.getPath(), childColExprPath,
+ childBaseTableExprPath);
+
+ putExprsIntoSmaps(analyzer, childSlotDesc, childColExprSlotRef,
+ childBaseTableExprSlotRef);
}
}
- return result;
+ }
+
+ // Verify that the paths belong to the same struct child.
+ private static void verifySameChild(Path childSlotDescPath, Path childColExprPath,
+ Path childBaseTableExprPath) {
+ List<String> childSlotDescRawPath = childSlotDescPath.getRawPath();
+ List<String> childColExprRawPath = childColExprPath.getRawPath();
+ List<String> childBaseTableExprRawPath = childBaseTableExprPath.getRawPath();
+
+ // Check that the children come in the same order for all of 'slotDesc', 'colExpr'
+ // and 'baseTableExpr'. If not, the last part of the paths would be different.
+ String childSlotDescPathEnd =
+ childSlotDescRawPath.get(childSlotDescRawPath.size() - 1);
+ String childColExprPathEnd =
+ childColExprRawPath.get(childColExprRawPath.size() - 1);
+ String childBaseTableExprPathEnd =
+ childBaseTableExprRawPath.get(childBaseTableExprRawPath.size() - 1);
+
+ Preconditions.checkState(childSlotDescPathEnd.equals(childColExprPathEnd));
+ Preconditions.checkState(childSlotDescPathEnd.equals(childBaseTableExprPathEnd));
}
/**
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
index 898059842..7ca2cf958 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
@@ -217,7 +217,8 @@ public class SlotDescriptor {
}
/**
- * Returns the slot descs of the structs that contain this slot desc, recursively.
+ * Returns the slot descs of the structs that contain this slot desc, recursively; stops
+ * at collection items, does not continue to the parent collection.
* For an example struct 'outer: <middle: <inner: <i: int>>>', called for the slot desc
* of 'i', the returned list will contain the slot descs of 'inner', 'middle' and
* 'outer'.
@@ -229,7 +230,8 @@ public class SlotDescriptor {
}
/**
- * Returns the tuple descs enclosing this slot desc, recursively.
+ * Returns the tuple descs enclosing this slot desc, recursively; stops at collection
+ * items, does not continue to the parent collection.
* For an example struct 'outer: <middle: <inner: <i: int>>>', called for the slot desc
* of 'i', the returned list will contain the 'itemTupleDesc_'s of 'inner', 'middle'
* and 'outer' as well as the tuple desc of the main tuple (the 'parent_' of the slot
@@ -241,6 +243,26 @@ public class SlotDescriptor {
return result;
}
+ /**
+ * Climbs up the struct hierarchy and returns the tuple desc that holds the top level
+ * struct that contains this slot desc; stops at collection items, does not continue to
+ * the parent collection.
+ * For a slot desc that is not within a struct, simply returns 'parent_'.
+ * For an example struct 'outer: <middle: <inner: <i: int>>>', called for the slot desc
+ * of 'i', returns the tuple desc of the main tuple (the 'parent_' of the slot desc of
+ * 'outer').
+ */
+ public TupleDescriptor getTopEnclosingTupleDesc() {
+ List<TupleDescriptor> enclosingTuples = getEnclosingTupleDescs();
+ if (enclosingTuples.isEmpty()) {
+ Preconditions.checkState(getParent() == null);
+ return null;
+ }
+
+ // Return the last enclosing tuple, going upward.
+ return enclosingTuples.get(enclosingTuples.size() - 1);
+ }
+
/**
* Returns the size of the slot without null indicators.
*
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index e9fa5d5ed..8f64b52c6 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -212,10 +212,6 @@ public class SlotRef extends Expr {
throw new AnalysisException("Unsupported type '"
+ fieldType.toSql() + "' in '" + toSql() + "'.");
}
- if (fieldType.isCollectionType()) {
- throw new AnalysisException("Struct containing a collection type is not " +
- "allowed in the select list.");
- }
if (fieldType.isBinary()) {
throw new AnalysisException("Struct containing a BINARY type is not " +
"allowed in the select list (IMPALA-11491).");
diff --git a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
index d5c42653f..04f522e5a 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
@@ -294,7 +294,7 @@ public class TupleDescriptor {
* Materialize all slots.
*/
public void materializeSlots() {
- for (SlotDescriptor slot: slots_) slot.setIsMaterialized(true);
+ for (SlotDescriptor slot: getSlotsRecursively()) slot.setIsMaterialized(true);
}
public TTupleDescriptor toThrift(Integer tableId) {
diff --git a/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java b/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java
index ec3355077..932a4fce6 100644
--- a/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java
@@ -20,6 +20,8 @@ package org.apache.impala.analysis;
import org.apache.impala.analysis.Path.PathType;
import org.apache.impala.analysis.TableRef.ZippingUnnestType;
import org.apache.impala.catalog.TableLoadingException;
+import org.apache.impala.catalog.ArrayType;
+import org.apache.impala.catalog.MapType;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
@@ -62,6 +64,12 @@ public class UnnestExpr extends SlotRef {
// find the corresponding CollectionTableRef during resolution.
Path resolvedPath = resolveAndVerifyRawPath(analyzer);
Preconditions.checkNotNull(resolvedPath);
+ verifyNotInsideStruct(resolvedPath);
+
+ Type type = resolvedPath.getMatchedTypes().get(
+ resolvedPath.getMatchedTypes().size() - 1);
+ verifyContainsNoStruct(type);
+
if (!rawPathWithoutItem_.isEmpty()) rawPathWithoutItem_.clear();
rawPathWithoutItem_.addAll(rawPath_);
@@ -83,6 +91,37 @@ public class UnnestExpr extends SlotRef {
analyzer.addZippingUnnestTupleId(desc_.getParent().getId());
}
+ // Verifies that the given path does not refer to a value that is within a struct.
+ // Note: If using the FROM clause zipping unnest syntax, this check has to be performed
+ // in CollectionTableRef.analyze().
+ public static void verifyNotInsideStruct(Path resolvedPath) throws AnalysisException {
+ for (Type type : resolvedPath.getMatchedTypes()) {
+ if (type.isStructType()) {
+ throw new AnalysisException(
+ "Zipping unnest on an array that is within a struct is not supported.");
+ }
+ }
+ }
+
+ // Verifies that the given type does not contain a struct. Descends along array items
+ // and map keys/values.
+ // Note: If using the FROM clause zipping unnest syntax, this check has to be performed
+ // in CollectionTableRef.analyze().
+ public static void verifyContainsNoStruct(Type type) throws AnalysisException {
+ if (type.isStructType()) {
+ throw new AnalysisException(
+ "Zipping unnest on an array that (recursively) " +
+ "contains a struct is not supported.");
+ } else if (type.isArrayType()) {
+ ArrayType arrType = (ArrayType) type;
+ verifyContainsNoStruct(arrType.getItemType());
+ } else if (type.isMapType()) {
+ MapType mapType = (MapType) type;
+ verifyContainsNoStruct(mapType.getKeyType());
+ verifyContainsNoStruct(mapType.getValueType());
+ }
+ }
+
private void verifyTableRefs(Analyzer analyzer) throws AnalysisException {
for (TableRef ref : analyzer.getTableRefs().values()) {
if (ref instanceof CollectionTableRef) {
@@ -210,4 +249,4 @@ public class UnnestExpr extends SlotRef {
}
return super.isBoundByTupleIds(tids);
}
-}
\ No newline at end of file
+}
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 7939e72c2..197f8731f 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -863,7 +863,12 @@ public class SingleNodePlanner {
SlotRef collectionExpr =
(SlotRef) collectionTableRef.getCollectionExpr();
if (collectionExpr != null) {
- requiredTids.add(collectionExpr.getDesc().getParent().getId());
+ // If the collection is within a (possibly nested) struct, add the tuple in
+ // which the top level struct is located.
+ SlotDescriptor desc = collectionExpr.getDesc();
+ List<TupleDescriptor> enclosingTupleDescs = desc.getEnclosingTupleDescs();
+ TupleDescriptor topTuple = desc.getTopEnclosingTupleDesc();
+ requiredTids.add(topTuple.getId());
} else {
requiredTids.add(collectionTableRef.getResolvedPath().getRootDesc().getId());
}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index ebad509d1..6e402a6a5 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -632,10 +632,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
testTableRefPath("select 1 from d.t7.c3.item.a2.item.a3", path(2, 0, 1, 0, 2), null);
testSlotRefPath("select item from d.t7.c3.a2.a3", path(2, 0, 1, 0, 2, 0));
testSlotRefPath("select item from d.t7.c3.item.a2.item.a3", path(2, 0, 1, 0, 2, 0));
- AnalysisContext ctx = createAnalysisCtx();
- ctx.getQueryOptions().setDisable_codegen(true);
- AnalysisError("select item from d.t7.c3", ctx,
- "Struct containing a collection type is not allowed in the select list.");
+ testSlotRefPath("select item from d.t7.c3", path(2, 0));
// Test path assembly with multiple tuple descriptors.
testTableRefPath("select 1 from d.t7, t7.c3, c3.a2, a2.a3",
path(2, 0, 1, 0, 2), path(2, 0, 1, 0, 2));
@@ -784,17 +781,16 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
"Querying STRUCT is only supported for ORC and Parquet file formats.");
AnalyzesOk("select alltypes from functional_orc_def.complextypes_structs", ctx);
- // Check if a struct in the select list raises an error if it contains collections.
+ // Check that a struct in the select list doesn't raise an error if it contains
+ // collections.
addTestTable(
"create table nested_structs (s1 struct<s2:struct<i:int>>) stored as orc");
addTestTable("create table nested_structs_with_list " +
"(s1 struct<s2:struct<a:array<int>>>) stored as orc");
AnalyzesOk("select s1 from nested_structs", ctx);
AnalyzesOk("select s1.s2 from nested_structs", ctx);
- AnalysisError("select s1 from nested_structs_with_list", ctx, "Struct containing " +
- "a collection type is not allowed in the select list.");
- AnalysisError("select s1.s2 from nested_structs_with_list", ctx, "Struct " +
- "containing a collection type is not allowed in the select list.");
+ AnalyzesOk("select s1 from nested_structs_with_list", ctx);
+ AnalyzesOk("select s1.s2 from nested_structs_with_list", ctx);
}
@Test
@@ -1088,19 +1084,15 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalyzesOk("select * from (select int_map, int_map_array from " +
"functional_parquet.complextypestbl) v",ctx);
- ctx.getQueryOptions().setDisable_codegen(false);
-
AnalyzesOk("select * from functional_parquet.complextypes_arrays",ctx);
AnalyzesOk("select * from " +
"functional_parquet.complextypes_arrays_only_view",ctx);
AnalyzesOk("select v.id, v.* from " +
"(select * from functional_parquet.complextypes_arrays) v",ctx);
- AnalysisError("select * from functional.allcomplextypes",
- ctx,"STRUCT type inside collection types is not supported.");
-
- AnalysisError("select * from functional_orc_def.complextypestbl", ctx,
- "Struct containing a collection type is not allowed in the select list.");
+ // Allow also structs in collections and vice versa.
+ AnalyzesOk("select * from functional_parquet.allcomplextypes", ctx);
+ AnalyzesOk("select * from functional_orc_def.complextypestbl", ctx);
AnalysisError("select * from functional_parquet.binary_in_complex_types", ctx,
"Binary type inside collection types is not supported (IMPALA-11491).");
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 27b7f3cee..68d5453dd 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -3783,6 +3783,7 @@ map_char_key MAP<CHAR(3), INT>
map_varchar_key MAP<VARCHAR(3), STRING>
map_timestamp_key MAP<TIMESTAMP, STRING>
map_date_key MAP<DATE, STRING>
+struct_contains_map STRUCT<m: MAP<INT, STRING>, s: STRING>
---- DEPENDENT_LOAD_HIVE
INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
(1,
@@ -3799,13 +3800,141 @@ INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
map(cast("a" as VARCHAR(3)), "A", if(false, cast("" as VARCHAR(3)), NULL), NULL),
map(to_utc_timestamp("2022-12-10 08:15:12", "UTC"), "Saturday morning",
if(false, to_utc_timestamp("2022-12-10 08:15:12", "UTC"), NULL), "null"),
- map(to_date("2022-12-10"), "Saturday", if(false, to_date("2022-12-10"), NULL), "null")
+ map(to_date("2022-12-10"), "Saturday", if(false, to_date("2022-12-10"), NULL), "null"),
+ named_struct("m", map(1, "one", if(false, 1, NULL), "null"), "s", "some_string")
);
---- LOAD
====
---- DATASET
functional
---- BASE_TABLE_NAME
+collection_struct_mix
+---- COLUMNS
+id INT
+struct_contains_arr STRUCT<arr: ARRAY<INT>>
+struct_contains_map STRUCT<m: MAP<INT, STRING>>
+arr_contains_struct ARRAY<STRUCT<i: BIGINT>>
+arr_contains_nested_struct ARRAY<STRUCT<inner_struct: STRUCT<str: STRING, l: INT>, small: SMALLINT>>
+struct_contains_nested_arr STRUCT<arr: ARRAY<ARRAY<DATE>>, i: INT>
+all_mix MAP<INT, STRUCT<big: STRUCT<arr: ARRAY<STRUCT<inner_arr: ARRAY<ARRAY<INT>>, m: TIMESTAMP>>, n: INT>, small: STRUCT<str: STRING, i: INT>>>
+---- DEPENDENT_LOAD_HIVE
+INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
+ (
+ 1,
+ named_struct("arr", array(1, 2, 3, 4, NULL, NULL, 5)),
+ named_struct("m", map(1, "one", 2, "two", 0, NULL)),
+ array(named_struct("i", 1L), named_struct("i", 2L), named_struct("i", 3L),
+ named_struct("i", 4L), NULL, named_struct("i", 5L), named_struct("i", NULL)),
+ array(named_struct("inner_struct", named_struct("str", "", "l", 0), "small", 2S), NULL,
+ named_struct("inner_struct", named_struct("str", "some_string", "l", 5), "small", 20S)),
+ named_struct("arr", array(array(to_date("2022-12-05"), to_date("2022-12-06"), NULL, to_date("2022-12-07")),
+ array(to_date("2022-12-08"), to_date("2022-12-09"), NULL)), "i", 2),
+ map(
+ 10,
+ named_struct(
+ "big", named_struct(
+ "arr", array(
+ named_struct(
+ "inner_arr", array(array(0, NULL, -1, -5, NULL, 8), array(20, NULL)),
+ "m", to_utc_timestamp("2022-12-05 14:30:00", "UTC")
+ ),
+ named_struct(
+ "inner_arr", array(array(12, 1024, NULL), array(NULL, NULL, 84), array(NULL, 15, NULL)),
+ "m", to_utc_timestamp("2022-12-06 16:20:52", "UTC")
+ )
+ ),
+ "n", 98
+ ),
+ "small", named_struct(
+ "str", "somestring",
+ "i", 100
+ )
+ )
+ )
+ ),
+ (
+ 2,
+ named_struct("arr", if(false, array(1), NULL)),
+ named_struct("m", if(false, map(1, "one"), NULL)),
+ array(named_struct("i", 100L), named_struct("i", 8L), named_struct("i", 35L),
+ named_struct("i", 45L), NULL, named_struct("i", 193L), named_struct("i", NULL)),
+ array(named_struct("inner_struct", if(false, named_struct("str", "", "l", 0), NULL), "small", 104S),
+ named_struct("inner_struct", named_struct("str", "aaa", "l", 28), "small", 105S), NULL),
+ named_struct("arr", array(array(to_date("2022-12-10"), to_date("2022-12-11"), NULL, to_date("2022-12-12")),
+ if(false, array(to_date("2022-12-12")), NULL)), "i", 2754),
+ map(
+ 20,
+ named_struct(
+ "big", named_struct(
+ "arr", array(
+ if(false, named_struct(
+ "inner_arr", array(array(0)),
+ "m", to_utc_timestamp("2022-12-10 08:01:05", "UTC")
+ ), NULL),
+ named_struct(
+ "inner_arr", array(array(12, 1024, NULL), array(NULL, NULL, 84), array(NULL, 15, NULL)),
+ "m", to_utc_timestamp("2022-12-10 08:15:12", "UTC")
+ )
+ ),
+ "n", 95
+ ),
+ "small", named_struct(
+ "str", "otherstring",
+ "i", 2048
+ )
+ ),
+ 21,
+ named_struct(
+ "big", named_struct(
+ "arr", if(false, array(
+ named_struct(
+ "inner_arr", array(array(0, NULL, -1, -5, NULL, 8), array(20, NULL)),
+ "m", to_utc_timestamp("2022-12-15 05:46:24", "UTC")
+ )
+ ), NULL),
+ "n", 8
+ ),
+ "small", named_struct(
+ "str", "textstring",
+ "i", 0
+ )
+ ),
+ 22,
+ named_struct(
+ "big", if(false, named_struct(
+ "arr", array(
+ named_struct(
+ "inner_arr", array(array(0)),
+ "m", if(false, to_utc_timestamp("2022-12-15 05:46:24", "UTC"), NULL)
+ )
+ ),
+ "n", 93
+ ), NULL),
+ "small", named_struct(
+ "str", "nextstring",
+ "i", 128
+ )
+ ),
+ 23,
+ NULL
+ )
+ );
+---- LOAD
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+collection_struct_mix_view
+---- CREATE
+SET disable_codegen=1;
+DROP VIEW IF EXISTS {db_name}{db_suffix}.{table_name};
+CREATE VIEW {db_name}{db_suffix}.{table_name}
+AS SELECT id, arr_contains_struct, arr_contains_nested_struct, struct_contains_nested_arr FROM {db_name}{db_suffix}.collection_struct_mix;
+---- LOAD
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
binary_tbl
---- COLUMNS
id INT
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index ad5abee48..6cf3b16a0 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -353,6 +353,12 @@ table_name:collection_tbl, constraint:restrict_to, table_format:orc/def/block
# In parquet we can't have NULL map keys but in ORC we can.
table_name:map_null_keys, constraint:restrict_to, table_format:orc/def/block
+table_name:collection_struct_mix, constraint:restrict_to, table_format:parquet/none/none
+table_name:collection_struct_mix, constraint:restrict_to, table_format:orc/def/block
+
+table_name:collection_struct_mix_view, constraint:restrict_to, table_format:parquet/none/none
+table_name:collection_struct_mix_view, constraint:restrict_to, table_format:orc/def/block
+
table_name:complextypes_maps_view, constraint:restrict_to, table_format:parquet/none/none
table_name:complextypes_maps_view, constraint:restrict_to, table_format:orc/def/block
diff --git a/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test b/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test
index 9d8cd8a89..184eb6b2a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test
@@ -15,12 +15,13 @@ select
map_char_key,
map_varchar_key,
map_timestamp_key,
- map_date_key
+ map_date_key,
+ struct_contains_map
from map_null_keys;
---- RESULTS
-1,'{true:"true",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1.75:"a",null:"null"}','{-1.75:"a",null:"null"}','{-1.8:"a",null:"null"}','{"one":1,null:null}','{"Mon":1,null:null}','{"a":"A",null:null}','{"2022-12-10 08:15:12":"Saturday morning",null:"null"}','{"2022-12-10":"Saturday",null:"null"}'
+1,'{true:"true",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1.75:"a",null:"null"}','{-1.75:"a",null:"null"}','{-1.8:"a",null:"null"}','{"one":1,null:null}','{"Mon":1,null:null}','{"a":"A",null:null}','{"2022-12-10 08:15:12":"Saturday morning",null:"null"}','{"2022-12-10":"Saturday",null:"null"}','{"m":{1:"one",null:"null"},"s":"some_string"}'
---- TYPES
-INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
+INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
=====
---- QUERY
-- Test that NULL map keys are printed correctly with STRINGIFY_MAP_KEYS=true.
@@ -39,10 +40,11 @@ select
map_char_key,
map_varchar_key,
map_timestamp_key,
- map_date_key
+ map_date_key,
+ struct_contains_map
from map_null_keys;
---- RESULTS
-1,'{"true":"true","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.8":"a","null":"null"}','{"one":1,"null":null}','{"Mon":1,"null":null}','{"a":"A","null":null}','{"2022-12-10 08:15:12":"Saturday morning","null":"null"}','{"2022-12-10":"Saturday","null":"null"}'
+1,'{"true":"true","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.8":"a","null":"null"}','{"one":1,"null":null}','{"Mon":1,"null":null}','{"a":"A","null":null}','{"2022-12-10 08:15:12":"Saturday morning","null":"null"}','{"2022-12-10":"Saturday","null":"null"}','{"m":{"1":"one","null":"null"},"s":"some_string"}'
---- TYPES
-INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
+INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
=====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test
new file mode 100644
index 000000000..56ba1c5d1
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test
@@ -0,0 +1,551 @@
+====
+---- QUERY
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select
+ id,
+ struct_contains_arr,
+ struct_contains_map,
+ arr_contains_struct,
+ arr_contains_nested_struct,
+ struct_contains_nested_arr,
+ all_mix
+from collection_struct_mix;
+---- RESULTS
+1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":" [...]
+2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"otherstring","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"str":"textstring","i":0}},22:{" [...]
+---- TYPES
+INT,STRING,STRING,STRING,STRING,STRING,STRING
+====
+---- QUERY
+select tbl.nested_struct from complextypestbl tbl;
+---- RESULTS
+'{"a":1,"b":[1],"c":{"d":[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]},"g":{"foo":{"h":{"i":[1.1]}}}}'
+'{"a":null,"b":[null],"c":{"d":[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]},"g":{"g1":{"h":{"i":[2.2,null]}},"g2":{"h":{"i":[]}},"g3":null,"g4":{"h":{"i":null}},"g5":{"h":null}}}'
+'{"a":null,"b":null,"c":{"d":[]},"g":{}}'
+'{"a":null,"b":null,"c":{"d":null},"g":null}'
+'{"a":null,"b":null,"c":null,"g":{"foo":{"h":{"i":[2.2,3.3]}}}}'
+'NULL'
+'{"a":7,"b":[2,3,null],"c":{"d":[[],[null],null]},"g":null}'
+'{"a":-1,"b":[-1],"c":{"d":[[{"e":-1,"f":"nonnullable"}]]},"g":{}}'
+---- TYPES
+STRING
+====
+---- QUERY
+select tbl.nested_struct.c from complextypestbl tbl;
+---- RESULTS
+'{"d":[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]}'
+'{"d":[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]}'
+'{"d":[]}'
+'{"d":null}'
+'NULL'
+'NULL'
+'{"d":[[],[null],null]}'
+'{"d":[[{"e":-1,"f":"nonnullable"}]]}'
+---- TYPES
+STRING
+====
+---- QUERY
+# Structs inside arrays are supported.
+select nested_struct.c.d from complextypestbl;
+---- RESULTS
+'[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]'
+'[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]'
+'[]'
+'NULL'
+'NULL'
+'NULL'
+'[[],[null],null]'
+'[[{"e":-1,"f":"nonnullable"}]]'
+---- TYPES
+STRING
+====
+---- QUERY
+# Structs inside maps are supported.
+select nested_struct.g from complextypestbl;
+---- RESULTS
+'{"foo":{"h":{"i":[1.1]}}}'
+'{"g1":{"h":{"i":[2.2,null]}},"g2":{"h":{"i":[]}},"g3":null,"g4":{"h":{"i":null}},"g5":{"h":null}}'
+'{}'
+'NULL'
+'{"foo":{"h":{"i":[2.2,3.3]}}}'
+'NULL'
+'NULL'
+'{}'
+---- TYPES
+STRING
+====
+---- QUERY
+# Select struct field from inline view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix)
+select sub.id, sub.struct_contains_nested_arr.arr from sub;
+---- RESULTS
+1,'[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]'
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select struct field from HMS view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, struct_contains_nested_arr.arr from collection_struct_mix_view;
+---- RESULTS
+1,'[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]'
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select array in struct from inline view and join-unnest it.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix)
+select id, arr.item from sub, sub.struct_contains_nested_arr.arr arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]'
+1,'["2022-12-08","2022-12-09",null]'
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select array in struct from HMS view and join-unnest it.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, arr.item from collection_struct_mix_view, collection_struct_mix_view.struct_contains_nested_arr.arr arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]'
+1,'["2022-12-08","2022-12-09",null]'
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select array in struct from nested inline view and join-unnest it.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix),
+ sub2 as (select id, struct_contains_nested_arr s from sub)
+select id, item from sub2, sub2.s.arr arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]'
+1,'["2022-12-08","2022-12-09",null]'
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Unnest an array that contains structs.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, item from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}'
+1,'NULL'
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}'
+2,'{"inner_struct":null,"small":104}'
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Unnest an array that contains structs from a nested inline view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, arr_contains_nested_struct arr from collection_struct_mix),
+ sub2 as (select id, arr arr2 from sub)
+select id, item from sub2, sub2.arr2 a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}'
+1,'NULL'
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}'
+2,'{"inner_struct":null,"small":104}'
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Unnest an array that contains structs from a HMS view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, item from collection_struct_mix_view,
+ collection_struct_mix_view.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}'
+1,'NULL'
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}'
+2,'{"inner_struct":null,"small":104}'
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Doubly unnest two-level array from nested inline view, displaying the unnested results
+# at both levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix),
+ sub2 as (select id, struct_contains_nested_arr s from sub)
+select id, arr.item, inner_arr.item from sub2, sub2.s.arr arr, arr.item inner_arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-05
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-06
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',NULL
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-07
+1,'["2022-12-08","2022-12-09",null]',2022-12-08
+1,'["2022-12-08","2022-12-09",null]',2022-12-09
+1,'["2022-12-08","2022-12-09",null]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-10
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-11
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-12
+---- TYPES
+INT,STRING,DATE
+====
+---- QUERY
+# Doubly unnest two-level array from HMS view, displaying the unnested results at both
+# levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, arr.item, inner_arr.item from collection_struct_mix_view,
+ collection_struct_mix_view.struct_contains_nested_arr.arr arr, arr.item inner_arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-05
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-06
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',NULL
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-07
+1,'["2022-12-08","2022-12-09",null]',2022-12-08
+1,'["2022-12-08","2022-12-09",null]',2022-12-09
+1,'["2022-12-08","2022-12-09",null]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-10
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-11
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-12
+---- TYPES
+INT,STRING,DATE
+====
+---- QUERY
+# Join unnest array containing struct and also query struct fields.
+select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix,
+ collection_struct_mix.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2
+1,'NULL','NULL',NULL
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20
+2,'{"inner_struct":null,"small":104}','NULL',104
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105
+2,'NULL','NULL',NULL
+---- TYPES
+INT,STRING,STRING,SMALLINT
+====
+---- QUERY
+# Join unnest array containing struct from HMS view and also query struct fields.
+select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix_view,
+ collection_struct_mix_view.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2
+1,'NULL','NULL',NULL
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20
+2,'{"inner_struct":null,"small":104}','NULL',104
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105
+2,'NULL','NULL',NULL
+---- TYPES
+INT,STRING,STRING,SMALLINT
+====
+---- QUERY
+# Join unnest array containing struct from inline view and also query struct fields.
+with sub as (select id, arr_contains_nested_struct from collection_struct_mix_view)
+select id, a.item, a.item.inner_struct, a.item.small from sub,
+ sub.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2
+1,'NULL','NULL',NULL
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20
+2,'{"inner_struct":null,"small":104}','NULL',104
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105
+2,'NULL','NULL',NULL
+---- TYPES
+INT,STRING,STRING,SMALLINT
+====
+---- QUERY
+# Zipping unnest an array that contains a struct.
+select unnest(arr_contains_struct) from collection_struct_mix;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest on an array that contains a struct with FROM-clause unnest syntax.
+select a.item from collection_struct_mix,
+ unnest(collection_struct_mix.arr_contains_struct) as (a);
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest of two arrays that contain a structs.
+select unnest(arr_contains_struct), unnest(arr_contains_nested_struct)
+from collection_struct_mix;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest of two arrays that contain a structs, from view
+select unnest(arr_contains_struct), unnest(arr_contains_nested_struct)
+from collection_struct_mix_view;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest of two arrays that contain structs, from view.
+with unnesting as (
+ select unnest(arr_contains_struct) struct1, unnest(arr_contains_nested_struct) struct2
+ from collection_struct_mix_view)
+select struct1, struct2 from unnesting
+where struct1.i > 2;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported.
+select unnest(struct_contains_nested_arr.arr) from collection_struct_mix;
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported with FROM-clause unnest
+# syntax.
+select a.item from collection_struct_mix,
+ unnest(collection_struct_mix.struct_contains_nested_arr.arr) as (a);
+---- RESULTS
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+---- TYPES
+STRING
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported; querying from a HMS
+# view.
+select unnest(struct_contains_nested_arr.arr) from collection_struct_mix_view;
+---- RESULTS
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+---- TYPES
+STRING
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported; querying from an inline
+# view.
+with sub as (select struct_contains_nested_arr from collection_struct_mix)
+select unnest(struct_contains_nested_arr.arr) from sub;
+---- RESULTS
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+---- TYPES
+STRING
+====
+---- QUERY
+# Test that NULL map keys are printed correctly with STRINGIFY_MAP_KEYS=true.
+set STRINGIFY_MAP_KEYS=1;
+select id, struct_contains_map from collection_struct_mix;
+---- RESULTS
+1,'{"m":{"1":"one","2":"two","0":null}}'
+2,'{"m":null}'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix_view)
+select id, struct_contains_nested_arr.arr from (select id, struct_contains_nested_arr from sub) sub2
+where struct_contains_nested_arr.i > 4;
+---- RESULTS
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix_view)
+select id, struct_contains_nested_arr.arr from
+ (select id, struct_contains_nested_arr
+ from sub
+ where struct_contains_nested_arr.i > 4) sub2;
+---- RESULTS
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, struct_contains_nested_arr
+ from collection_struct_mix_view
+ where struct_contains_nested_arr.i > 4)
+select id, struct_contains_nested_arr.arr from (select id, struct_contains_nested_arr from
+sub) sub2;
+---- RESULTS
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, struct_contains_nested_arr
+ from collection_struct_mix_view)
+select id, arr2.item from (
+ select id, arr.item single_arr
+ from sub, sub.struct_contains_nested_arr.arr arr) sub2, sub2.single_arr arr2
+where arr2.item > "2022-12-06";
+---- RESULTS
+1,2022-12-07
+1,2022-12-08
+1,2022-12-09
+2,2022-12-10
+2,2022-12-11
+2,2022-12-12
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, struct_contains_nested_arr
+ from collection_struct_mix_view)
+select id, arr2.item from (
+ select id, arr.item single_arr
+ from sub, sub.struct_contains_nested_arr.arr arr
+ where sub.struct_contains_nested_arr.i > 4) sub2, sub2.single_arr arr2
+where arr2.item > "2022-12-06";
+---- RESULTS
+2,2022-12-10
+2,2022-12-11
+2,2022-12-12
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, arr1.item single_arr
+ from collection_struct_mix, collection_struct_mix.struct_contains_nested_arr.arr arr1
+ where struct_contains_nested_arr.i > 4)
+select id, d from (
+ select id, arr2.item d
+ from sub, sub.single_arr arr2) sub2
+where d > "2022-12-06";
+---- RESULTS
+2,2022-12-10
+2,2022-12-11
+2,2022-12-12
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Using WHERE filters on the same column at different view levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with project as (
+ select id, struct_contains_nested_arr.arr nested_array from collection_struct_mix_view
+),
+unnest_to_1d_array as (
+ select id, item array_1d from project, project.nested_array
+),
+unnest_to_scalars as (
+ select id, item scalar_item from unnest_to_1d_array, unnest_to_1d_array.array_1d
+ where item > "2022-12-05"
+)
+select id, scalar_item from unnest_to_scalars
+where scalar_item < "2022-12-11";
+---- RESULTS
+1,2022-12-06
+1,2022-12-07
+1,2022-12-08
+1,2022-12-09
+2,2022-12-10
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Test that complex types are propagated through views with EXPAND_COMPLEX_TYPES=1.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=1;
+with sub as (
+ select * from collection_struct_mix)
+select * from (
+ select * from sub
+) sub2;
+---- RESULTS
+1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":" [...]
+2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"otherstring","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"str":"textstring","i":0}},22:{" [...]
+---- TYPES
+INT,STRING,STRING,STRING,STRING,STRING,STRING
+====
+---- QUERY
+# Test that struct elements can be star-exanded with EXPAND_COMPLEX_TYPES=0;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=0;
+with sub as (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from
+collection_struct_mix)
+select id, arr_contains_nested_struct, struct_contains_nested_arr.* from (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from sub
+) sub2;
+---- RESULTS
+1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]',2
+2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]',2754
+---- TYPES
+INT,STRING,INT
+====
+---- QUERY
+# Test that struct elements can be star-exanded with EXPAND_COMPLEX_TYPES=1;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=1;
+with sub as (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from
+collection_struct_mix)
+select id, arr_contains_nested_struct, struct_contains_nested_arr.* from (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from sub
+) sub2;
+---- RESULTS
+1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]',2
+2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','[["2022-12-10","2022-12-11",null,"2022-12-12"],null]',2754
+---- TYPES
+INT,STRING,STRING,INT
+====
+---- QUERY
+# Test that struct elements from unnested array can be star-exanded with
+# EXPAND_COMPLEX_TYPES=0;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=0;
+select id, arr.item.*
+from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr;
+---- RESULTS
+1,2
+1,NULL
+1,20
+2,104
+2,105
+2,NULL
+---- TYPES
+INT,SMALLINT
+====
+---- QUERY
+# Test that struct elements from unnested array can be star-exanded with
+# EXPAND_COMPLEX_TYPES=1;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=1;
+select id, arr.item.*
+from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr;
+---- RESULTS
+1,'{"str":"","l":0}',2
+1,'NULL',NULL
+1,'{"str":"some_string","l":5}',20
+2,'NULL',104
+2,'{"str":"aaa","l":28}',105
+2,'NULL',NULL
+---- TYPES
+INT,STRING,SMALLINT
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
index 6e7b57935..24bf1b20e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
@@ -79,7 +79,7 @@ INT
set EXPAND_COMPLEX_TYPES=1;
select nested_struct.* from complextypestbl
---- CATCH
-AnalysisException: Struct containing a collection type is not allowed in the select list.
+AnalysisException: Struct type in select list is not allowed when Codegen is ON. You might want to set DISABLE_CODEGEN=true
====
---- QUERY
# Test resolving explicit STAR path on a nested struct column inside array
diff --git a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
index f2b5ed498..082892649 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
@@ -597,16 +597,6 @@ where alltypes in (select alltypes from functional_parquet.complextypes_structs)
AnalysisException: A subquery can't return complex types. (SELECT alltypes FROM functional_parquet.complextypes_structs)
====
---- QUERY
-select tbl.nested_struct from complextypestbl tbl;
----- CATCH
-AnalysisException: Struct containing a collection type is not allowed in the select list.
-====
----- QUERY
-select tbl.nested_struct.c from complextypestbl tbl;
----- CATCH
-AnalysisException: Struct containing a collection type is not allowed in the select list.
-====
----- QUERY
# Unioning structs is not supported.
# IMPALA-10752
select id, tiny_struct from complextypes_structs
@@ -636,15 +626,3 @@ order by 3
---- CATCH
AnalysisException: ORDER BY: ordinal exceeds the number of items in the SELECT list: 3
====
----- QUERY
-# Structs inside arrays are not yet supported.
-select nested_struct.c.d from complextypestbl;
----- CATCH
-AnalysisException: STRUCT type inside collection types is not supported.
-====
----- QUERY
-# Structs inside maps are not yet supported.
-select nested_struct.g from complextypestbl;
----- CATCH
-AnalysisException: STRUCT type inside collection types is not supported.
-====
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 999989440..99d44e537 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -182,9 +182,41 @@ class TestNestedCollectionsInSelectList(ImpalaTestSuite):
"""Queries where a map has null keys. Is only possible in ORC, not Parquet."""
if vector.get_value('table_format').file_format == 'parquet':
pytest.skip()
+ # Structs in select list are not supported with codegen enabled: see IMPALA-10851.
+ if vector.get_value('exec_option')['disable_codegen'] == 'False':
+ pytest.skip()
self.run_test_case('QueryTest/map_null_keys', vector)
+class TestMixedCollectionsAndStructsInSelectList(ImpalaTestSuite):
+ """Functional tests for the case where collections and structs are embedded into one
+ another and they are provided in the select list."""
+ @classmethod
+ def get_workload(self):
+ return 'functional-query'
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestMixedCollectionsAndStructsInSelectList, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ v.get_value('table_format').file_format in ['parquet', 'orc'])
+ cls.ImpalaTestMatrix.add_dimension(
+ ImpalaTestDimension('mt_dop', 0, 2))
+ cls.ImpalaTestMatrix.add_dimension(
+ create_exec_option_dimension_from_dict({
+ 'disable_codegen': ['False', 'True']}))
+ cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
+ cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('orc_schema_resolution', 0, 1))
+ cls.ImpalaTestMatrix.add_constraint(orc_schema_resolution_constraint)
+
+ def test_mixed_complex_types_in_select_list(self, vector, unique_database):
+ """Queries where structs and collections are embedded into one another."""
+ # Structs in select list are not supported with codegen enabled: see IMPALA-10851.
+ if vector.get_value('exec_option')['disable_codegen'] == 'False':
+ pytest.skip()
+ self.run_test_case('QueryTest/mixed-collections-and-structs', vector)
+
+
class TestComputeStatsWithNestedTypes(ImpalaTestSuite):
"""Functional tests for running compute stats on tables that have nested types in the
columns."""