You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by db...@apache.org on 2023/03/07 13:49:03 UTC
[impala] branch master updated (67bb870aa -> 2d4730698)
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 67bb870aa IMPALA-11911: Fix NULL argument handling in Hive GenericUDFs
new 374d011a7 IMPALA-11479: Add Java unit tests for IcebergUtil.
new d98ab986a IMPALA-11223: Use unique id to create codegen instances
new 2d4730698 IMPALA-9551: Allow mixed complex types in select list
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/codegen/codegen-symbol-emitter.cc | 5 +-
be/src/codegen/llvm-codegen.cc | 2 +
be/src/exec/unnest-node.cc | 9 +-
be/src/exprs/slot-ref.h | 1 +
be/src/runtime/complex-value-writer.h | 6 +-
be/src/runtime/complex-value-writer.inline.h | 74 ++-
be/src/runtime/descriptors.cc | 12 +-
be/src/runtime/fragment-state.cc | 6 +-
be/src/runtime/raw-value.cc | 10 +-
be/src/service/hs2-util.cc | 23 +-
be/src/service/query-result-set.cc | 7 +-
.../java/org/apache/impala/analysis/Analyzer.java | 201 +++++---
.../apache/impala/analysis/CollectionTableRef.java | 8 +
.../org/apache/impala/analysis/InlineViewRef.java | 229 +++++----
.../org/apache/impala/analysis/SlotDescriptor.java | 26 +-
.../java/org/apache/impala/analysis/SlotRef.java | 4 -
.../apache/impala/analysis/TupleDescriptor.java | 2 +-
.../org/apache/impala/analysis/UnnestExpr.java | 41 +-
.../apache/impala/common/TransactionKeepalive.java | 1 -
.../apache/impala/planner/SingleNodePlanner.java | 7 +-
.../java/org/apache/impala/util/IcebergUtil.java | 113 +++--
.../apache/impala/analysis/AnalyzeStmtsTest.java | 24 +-
.../impala/catalog/local/LocalCatalogTest.java | 2 +-
.../org/apache/impala/util/IcebergUtilTest.java | 400 +++++++++++++++
.../functional/functional_schema_template.sql | 131 ++++-
.../datasets/functional/schema_constraints.csv | 6 +
.../queries/QueryTest/map_null_keys.test | 14 +-
.../QueryTest/mixed-collections-and-structs.test | 551 +++++++++++++++++++++
.../ranger_column_masking_complex_types.test | 2 +-
.../queries/QueryTest/struct-in-select-list.test | 22 -
tests/query_test/test_nested_types.py | 32 ++
31 files changed, 1657 insertions(+), 314 deletions(-)
create mode 100644 fe/src/test/java/org/apache/impala/util/IcebergUtilTest.java
create mode 100644 testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test
[impala] 02/03: IMPALA-11223: Use unique id to create codegen instances
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit d98ab986a6a2218523ec147f70110300f019150b
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Thu Feb 16 20:30:50 2023 +0800
IMPALA-11223: Use unique id to create codegen instances
When startup flag asm_module_dir is set, impalad will dump the codegen
disassembly to files under that folder. The file name is "id.asm" in
which "id" is the codegen instance id. Before IMPALA-4080 (f2837e9), we
used fragment instance id as the codegen id. After that, since codegen
is done in fragment level (shared by fragment instances), we use query
id instead. This introduces conflicts between different fragments. The
asm files will be overwritten.
The same conflict happens in dumping IR modules (when unopt_module_dir
or opt_module_dir is set).
This changes the codegen instance id to be "QueryID_FragmentName_PID".
The PID suffix is needed since we usually have several impalads running
together on our dev box.
Also adds logs when IR or disassembly are dumped to files. It helps to
know which instance performs the codegen.
Tests:
- Manually verified the asm file names are expected.
Change-Id: I7672906365c916bbe750eeb9906cab38573e6c31
Reviewed-on: http://gerrit.cloudera.org:8080/19505
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/codegen/codegen-symbol-emitter.cc | 5 ++++-
be/src/codegen/llvm-codegen.cc | 2 ++
be/src/runtime/fragment-state.cc | 6 +++++-
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/be/src/codegen/codegen-symbol-emitter.cc b/be/src/codegen/codegen-symbol-emitter.cc
index 992493bff..ce8d3c547 100644
--- a/be/src/codegen/codegen-symbol-emitter.cc
+++ b/be/src/codegen/codegen-symbol-emitter.cc
@@ -74,7 +74,10 @@ void CodegenSymbolEmitter::NotifyObjectEmitted(const llvm::object::ObjectFile& o
ProcessSymbol(&dwarf_ctx, pair.first, pair.second, &perf_map_entries, asm_file);
}
- if (asm_file.is_open()) asm_file.close();
+ if (asm_file.is_open()) {
+ asm_file.close();
+ LOG(INFO) << "Saved disassembly to " << asm_path_;
+ }
ofstream perf_map_file;
if (emit_perf_map_) {
diff --git a/be/src/codegen/llvm-codegen.cc b/be/src/codegen/llvm-codegen.cc
index d31eee772..1b3b835ff 100644
--- a/be/src/codegen/llvm-codegen.cc
+++ b/be/src/codegen/llvm-codegen.cc
@@ -1262,6 +1262,7 @@ Status LlvmCodeGen::FinalizeModule() {
} else {
f << GetIR(true);
f.close();
+ LOG(INFO) << "Saved unoptimized IR to " << path;
}
}
@@ -1328,6 +1329,7 @@ Status LlvmCodeGen::FinalizeModule() {
} else {
f << GetIR(true);
f.close();
+ LOG(INFO) << "Saved optimized IR to " << path;
}
}
diff --git a/be/src/runtime/fragment-state.cc b/be/src/runtime/fragment-state.cc
index 60a275176..a9e8d23b5 100644
--- a/be/src/runtime/fragment-state.cc
+++ b/be/src/runtime/fragment-state.cc
@@ -155,8 +155,12 @@ void FragmentState::ReleaseResources() {
Status FragmentState::CreateCodegen() {
if (codegen_.get() != NULL) return Status::OK();
+ // Create with an id of "QueryId_FragmentName_PID" to avoid conflicts between fragments.
+ // The id is used when dumping IR modules or codegen disassembly.
RETURN_IF_ERROR(LlvmCodeGen::CreateImpalaCodegen(
- this, query_mem_tracker(), PrintId(query_id()), &codegen_));
+ this, query_mem_tracker(),
+ Substitute("$0_$1_$2", PrintId(query_id()), fragment_.display_name, getpid()),
+ &codegen_));
codegen_->EnableOptimizations(true);
runtime_profile_->AddChild(codegen_->runtime_profile());
return Status::OK();
[impala] 03/03: IMPALA-9551: Allow mixed complex types in select list
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2d47306987774d5fd134c798183b97dd81d26340
Author: Daniel Becker <da...@cloudera.com>
AuthorDate: Fri Oct 21 11:03:30 2022 +0200
IMPALA-9551: Allow mixed complex types in select list
Currently collections and structs are supported in the select list, also
when they are nested (structs in structs and collections in
collections), but mixing different kinds of complex types, i.e. having
structs in collections or vice versa, is not supported.
This patch adds support for mixed complex types in the select list.
Limitation: zipping unnest is not supported for mixed complex types, for
example the following query:
use functional_parquet;
select unnest(struct_contains_nested_arr.arr) from
collection_struct_mix;
Testing:
- Created a new test table, 'collection_struct_mix', that contains
mixed complex types.
- Added tests in mixed-collections-and-structs.test that test having
mixed complex types in the select list. These tests are called from
test_nested_types.py::TestMixedCollectionsAndStructsInSelectList.
- Ran existing tests that test collections and structs in the select
list; test queries that expected a failure in case of mixed complex
types have been moved to mixed-collections-and-structs.test and now
expect success.
Change-Id: I476d98884b5fd192dfcd4feeec7947526aebe993
Reviewed-on: http://gerrit.cloudera.org:8080/19322
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exec/unnest-node.cc | 9 +-
be/src/exprs/slot-ref.h | 1 +
be/src/runtime/complex-value-writer.h | 6 +-
be/src/runtime/complex-value-writer.inline.h | 74 ++-
be/src/runtime/descriptors.cc | 12 +-
be/src/runtime/raw-value.cc | 10 +-
be/src/service/hs2-util.cc | 23 +-
be/src/service/query-result-set.cc | 7 +-
.../java/org/apache/impala/analysis/Analyzer.java | 201 +++++---
.../apache/impala/analysis/CollectionTableRef.java | 8 +
.../org/apache/impala/analysis/InlineViewRef.java | 229 +++++----
.../org/apache/impala/analysis/SlotDescriptor.java | 26 +-
.../java/org/apache/impala/analysis/SlotRef.java | 4 -
.../apache/impala/analysis/TupleDescriptor.java | 2 +-
.../org/apache/impala/analysis/UnnestExpr.java | 41 +-
.../apache/impala/planner/SingleNodePlanner.java | 7 +-
.../apache/impala/analysis/AnalyzeStmtsTest.java | 24 +-
.../functional/functional_schema_template.sql | 131 ++++-
.../datasets/functional/schema_constraints.csv | 6 +
.../queries/QueryTest/map_null_keys.test | 14 +-
.../QueryTest/mixed-collections-and-structs.test | 551 +++++++++++++++++++++
.../ranger_column_masking_complex_types.test | 2 +-
.../queries/QueryTest/struct-in-select-list.test | 22 -
tests/query_test/test_nested_types.py | 32 ++
24 files changed, 1186 insertions(+), 256 deletions(-)
diff --git a/be/src/exec/unnest-node.cc b/be/src/exec/unnest-node.cc
index 36f084d72..f48596cab 100644
--- a/be/src/exec/unnest-node.cc
+++ b/be/src/exec/unnest-node.cc
@@ -62,7 +62,14 @@ Status UnnestPlanNode::InitCollExprs(FragmentState* state) {
SlotDescriptor* slot_desc = state->desc_tbl().GetSlotDescriptor(slot_ref->slot_id());
DCHECK(slot_desc != nullptr);
coll_slot_descs_.push_back(slot_desc);
- coll_tuple_idxs_.push_back(row_desc.GetTupleIdx(slot_desc->parent()->id()));
+
+ // If the collection is in a struct we don't use the itemTupleDesc of the struct but
+ // the tuple in which the top level struct is placed.
+ const TupleDescriptor* parent_tuple = slot_desc->parent();
+ const TupleDescriptor* master_tuple = parent_tuple->getMasterTuple();
+ const TupleDescriptor* top_level_tuple = master_tuple == nullptr ?
+ parent_tuple : master_tuple;
+ coll_tuple_idxs_.push_back(row_desc.GetTupleIdx(top_level_tuple->id()));
}
return Status::OK();
}
diff --git a/be/src/exprs/slot-ref.h b/be/src/exprs/slot-ref.h
index c0b1bf31a..7ebc1251d 100644
--- a/be/src/exprs/slot-ref.h
+++ b/be/src/exprs/slot-ref.h
@@ -63,6 +63,7 @@ class SlotRef : public ScalarExpr {
const SlotId& slot_id() const { return slot_id_; }
static const char* LLVM_CLASS_NAME;
NullIndicatorOffset GetNullIndicatorOffset() const { return null_indicator_offset_; }
+ const SlotDescriptor* GetSlotDescriptor() const { return slot_desc_; }
int GetSlotOffset() const { return slot_offset_; }
virtual const TupleDescriptor* GetCollectionTupleDesc() const override;
diff --git a/be/src/runtime/complex-value-writer.h b/be/src/runtime/complex-value-writer.h
index e1c7d05fc..5a85bbeb7 100644
--- a/be/src/runtime/complex-value-writer.h
+++ b/be/src/runtime/complex-value-writer.h
@@ -44,15 +44,17 @@ class ComplexValueWriter {
void CollectionValueToJSON(const CollectionValue& collection_value,
PrimitiveType collection_type, const TupleDescriptor* item_tuple_desc);
- // Gets a non-null StructVal and writes it in JSON format. Uses 'column_type' to figure
+ // Gets a non-null StructVal and writes it in JSON format. Uses 'slot_desc' to figure
// out field names and types. This function can call itself recursively in case of
// nested structs.
void StructValToJSON(const impala_udf::StructVal& struct_val,
- const ColumnType& column_type);
+ const SlotDescriptor& slot_desc);
private:
void PrimitiveValueToJSON(void* value, const ColumnType& type, bool map_key);
void WriteNull(bool map_key);
+ void StructInCollectionToJSON(Tuple* item_tuple,
+ const SlotDescriptor& struct_slot_desc);
void CollectionElementToJSON(Tuple* item_tuple, const SlotDescriptor& slot_desc,
bool map_key);
void ArrayValueToJSON(const CollectionValue& array_value,
diff --git a/be/src/runtime/complex-value-writer.inline.h b/be/src/runtime/complex-value-writer.inline.h
index e73a17b60..c53325e1d 100644
--- a/be/src/runtime/complex-value-writer.inline.h
+++ b/be/src/runtime/complex-value-writer.inline.h
@@ -81,6 +81,51 @@ void ComplexValueWriter<JsonStream>::WriteNull(bool map_key) {
}
}
+// Structs in collections are not converted to StructVal but are left as they are in the
+// tuple.
+template <class JsonStream>
+void ComplexValueWriter<JsonStream>::StructInCollectionToJSON(Tuple* item_tuple,
+ const SlotDescriptor& struct_slot_desc) {
+ const TupleDescriptor* children_tuple_desc =
+ struct_slot_desc.children_tuple_descriptor();
+ DCHECK(children_tuple_desc != nullptr);
+
+ const ColumnType& struct_type = struct_slot_desc.type();
+ const std::vector<SlotDescriptor*>& child_slots = children_tuple_desc->slots();
+
+ DCHECK(struct_type.type == TYPE_STRUCT);
+ DCHECK_EQ(child_slots.size(), struct_type.children.size());
+
+ writer_->StartObject();
+ for (int i = 0; i < child_slots.size(); ++i) {
+ writer_->String(struct_type.field_names[i].c_str());
+
+ const SlotDescriptor& child_slot_desc = *child_slots[i];
+ bool element_is_null = item_tuple->IsNull(child_slot_desc.null_indicator_offset());
+ if (element_is_null) {
+ WriteNull(writer_);
+ continue;
+ }
+
+ const ColumnType& child_type = child_slot_desc.type();
+ void* child = item_tuple->GetSlot(child_slot_desc.tuple_offset());
+ if (child_type.IsStructType()) {
+ StructInCollectionToJSON(item_tuple, child_slot_desc);
+ } else if (child_type.IsCollectionType()) {
+ const CollectionValue* nested_collection_val =
+ reinterpret_cast<CollectionValue*>(child);
+ const TupleDescriptor* child_item_tuple_desc =
+ child_slot_desc.children_tuple_descriptor();
+ DCHECK(child_item_tuple_desc != nullptr);
+ CollectionValueToJSON(*nested_collection_val, child_type.type,
+ child_item_tuple_desc);
+ } else {
+ PrimitiveValueToJSON(child, child_type, false);
+ }
+ }
+ writer_->EndObject();
+}
+
template <class JsonStream>
void ComplexValueWriter<JsonStream>::CollectionElementToJSON(Tuple* item_tuple,
const SlotDescriptor& slot_desc, bool map_key) {
@@ -91,7 +136,8 @@ void ComplexValueWriter<JsonStream>::CollectionElementToJSON(Tuple* item_tuple,
if (element_is_null) {
WriteNull(map_key);
} else if (element_type.IsStructType()) {
- DCHECK(false) << "Structs in collections are not supported yet.";
+ DCHECK(!map_key) << "Structs cannot be map keys.";
+ StructInCollectionToJSON(item_tuple, slot_desc);
} else if (element_type.IsCollectionType()) {
const CollectionValue* nested_collection_val =
reinterpret_cast<CollectionValue*>(element);
@@ -179,18 +225,32 @@ void ComplexValueWriter<JsonStream>::CollectionValueToJSON(
template <class JsonStream>
void ComplexValueWriter<JsonStream>::StructValToJSON(const StructVal& struct_val,
- const ColumnType& column_type) {
- DCHECK(column_type.type == TYPE_STRUCT);
- DCHECK_EQ(struct_val.num_children, column_type.children.size());
+ const SlotDescriptor& slot_desc) {
+ const ColumnType& struct_type = slot_desc.type();
+ DCHECK(struct_type.type == TYPE_STRUCT);
+ DCHECK_EQ(struct_val.num_children, struct_type.children.size());
+
+ const TupleDescriptor* children_item_tuple_desc = slot_desc.children_tuple_descriptor();
+ DCHECK(children_item_tuple_desc != nullptr);
+ const std::vector<SlotDescriptor*>& child_slot_descs =
+ children_item_tuple_desc->slots();
+ DCHECK_EQ(struct_val.num_children, child_slot_descs.size());
+
writer_->StartObject();
for (int i = 0; i < struct_val.num_children; ++i) {
- writer_->String(column_type.field_names[i].c_str());
+ writer_->String(struct_type.field_names[i].c_str());
void* child = (void*)(struct_val.ptr[i]);
- const ColumnType& child_type = column_type.children[i];
+ const SlotDescriptor& child_slot_desc = *child_slot_descs[i];
+ const ColumnType& child_type = struct_type.children[i];
+ DCHECK_EQ(child_type, child_slot_desc.type());
if (child == nullptr) {
WriteNull(false);
} else if (child_type.IsStructType()) {
- StructValToJSON(*((StructVal*)child), child_type);
+ StructValToJSON(*((StructVal*)child), child_slot_desc);
+ } else if (child_type.IsCollectionType()) {
+ CollectionValue* collection_child = reinterpret_cast<CollectionValue*>(child);
+ ComplexValueWriter::CollectionValueToJSON(*collection_child, child_type.type,
+ child_slot_desc.children_tuple_descriptor());
} else {
PrimitiveValueToJSON(child, child_type, false);
}
diff --git a/be/src/runtime/descriptors.cc b/be/src/runtime/descriptors.cc
index 2b3ffc92f..47234e168 100644
--- a/be/src/runtime/descriptors.cc
+++ b/be/src/runtime/descriptors.cc
@@ -352,17 +352,17 @@ TupleDescriptor::TupleDescriptor(const TTupleDescriptor& tdesc)
void TupleDescriptor::AddSlot(SlotDescriptor* slot) {
slots_.push_back(slot);
+ // If this is a tuple for struct children then we populate the 'string_slots_' field (in
+ // case of a var len string type) or the 'collection_slots_' field (in case of a
+ // collection type) of the topmost tuple and not this one.
+ TupleDescriptor* const target_tuple = isTupleOfStructSlot() ? master_tuple_ : this;
if (slot->type().IsVarLenStringType()) {
- TupleDescriptor* target_tuple = this;
- // If this is a tuple for struct children then we populate the 'string_slots_' of
- // the topmost tuple and not this one.
- if (isTupleOfStructSlot()) target_tuple = master_tuple_;
target_tuple->string_slots_.push_back(slot);
target_tuple->has_varlen_slots_ = true;
}
if (slot->type().IsCollectionType()) {
- collection_slots_.push_back(slot);
- has_varlen_slots_ = true;
+ target_tuple->collection_slots_.push_back(slot);
+ target_tuple->has_varlen_slots_ = true;
}
}
diff --git a/be/src/runtime/raw-value.cc b/be/src/runtime/raw-value.cc
index 32ca90e04..be8058db0 100644
--- a/be/src/runtime/raw-value.cc
+++ b/be/src/runtime/raw-value.cc
@@ -301,7 +301,7 @@ void RawValue::PrintValue(
case TYPE_BOOLEAN: {
bool val = *reinterpret_cast<const bool*>(value);
*stream << (val ? "true" : "false");
- return;
+ break;
}
case TYPE_TINYINT:
// Extra casting for chars since they should not be interpreted as ASCII.
@@ -410,12 +410,4 @@ template void RawValue::WritePrimitive<true>(const void* value, Tuple* tuple,
template void RawValue::WritePrimitive<false>(const void* value, Tuple* tuple,
const SlotDescriptor* slot_desc, MemPool* pool,
std::vector<StringValue*>* string_values);
-
-bool PrintNestedValueIfNull(const SlotDescriptor& slot_desc, Tuple* item,
- stringstream* stream) {
- bool is_null = item->IsNull(slot_desc.null_indicator_offset());
- if (is_null) *stream << RawValue::NullLiteral(false);
- return is_null;
-}
-
}
diff --git a/be/src/service/hs2-util.cc b/be/src/service/hs2-util.cc
index b3255d545..20279699c 100644
--- a/be/src/service/hs2-util.cc
+++ b/be/src/service/hs2-util.cc
@@ -26,6 +26,7 @@
#include "common/logging.h"
#include "exprs/scalar-expr.h"
#include "exprs/scalar-expr-evaluator.h"
+#include "exprs/slot-ref.h"
#include "gen-cpp/TCLIService_constants.h"
#include "runtime/date-value.h"
#include "runtime/complex-value-writer.inline.h"
@@ -371,7 +372,8 @@ static void DecimalExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
static void StructExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
const TColumnType& type, RowBatch* batch, int start_idx, int num_rows,
- uint32_t output_row_idx, apache::hive::service::cli::thrift::TColumn* column) {
+ uint32_t output_row_idx, bool stringify_map_keys,
+ apache::hive::service::cli::thrift::TColumn* column) {
DCHECK(type.types.size() > 1);
ReserveSpace(num_rows, output_row_idx, &column->stringVal);
// The buffer used by rapidjson::Writer. We reuse it to eliminate allocations.
@@ -381,16 +383,19 @@ static void StructExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
if (struct_val.is_null) {
column->stringVal.values.emplace_back();
} else {
- int idx = 0;
- ColumnType column_type(type.types, &idx);
+ const impala::ScalarExpr& scalar_expr = expr_eval->root();
+ // Currently scalar_expr can be only a slot ref as no functions return arrays.
+ DCHECK(scalar_expr.IsSlotRef());
+ const SlotDescriptor* slot_desc =
+ static_cast<const SlotRef&>(scalar_expr).GetSlotDescriptor();
+ DCHECK(slot_desc != nullptr);
buffer.Clear();
rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
- // TODO: Create a stringify_map_keys parameter and pass that when
- // "IMPALA-9551: Allow mixed complex types in select list" is done.
- ComplexValueWriter<rapidjson::StringBuffer> complex_value_writer(&writer, false);
- complex_value_writer.StructValToJSON(struct_val, column_type);
+ ComplexValueWriter<rapidjson::StringBuffer> complex_value_writer(&writer,
+ stringify_map_keys);
+ complex_value_writer.StructValToJSON(struct_val, *slot_desc);
column->stringVal.values.emplace_back(buffer.GetString());
}
@@ -451,8 +456,8 @@ void impala::ExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
// to inline the expression evaluation into the loop body.
switch (type.types[0].type) {
case TTypeNodeType::STRUCT:
- StructExprValuesToHS2TColumn(
- expr_eval, type, batch, start_idx, num_rows, output_row_idx, column);
+ StructExprValuesToHS2TColumn(expr_eval, type, batch, start_idx, num_rows,
+ output_row_idx, stringify_map_keys, column);
return;
case TTypeNodeType::ARRAY:
case TTypeNodeType::MAP:
diff --git a/be/src/service/query-result-set.cc b/be/src/service/query-result-set.cc
index e90efe7ea..dd7a2b2f0 100644
--- a/be/src/service/query-result-set.cc
+++ b/be/src/service/query-result-set.cc
@@ -252,9 +252,14 @@ void QueryResultSet::PrintComplexValue(ScalarExprEvaluator* expr_eval,
} else {
DCHECK(type.IsStructType());
const StructVal* struct_val = static_cast<const StructVal*>(value);
+ const SlotDescriptor* slot_desc =
+ static_cast<const SlotRef&>(scalar_expr).GetSlotDescriptor();
+ DCHECK(slot_desc != nullptr);
+ DCHECK_EQ(type, slot_desc->type());
+
ComplexValueWriter<rapidjson::BasicOStreamWrapper<stringstream>>
complex_value_writer(&writer, stringify_map_keys);
- complex_value_writer.StructValToJSON(*struct_val, type);
+ complex_value_writer.StructValToJSON(*struct_val, *slot_desc);
}
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 2acdea7a9..b5945eb76 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -17,10 +17,12 @@
package org.apache.impala.analysis;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
@@ -238,6 +240,48 @@ public class Analyzer {
// to be applied of the Iceberg V2 table.
private long totalRecordsNumV2_;
+ // The method 'registerSlotRef()' is called recursively for complex types (structs and
+ // collections). When creating new 'SlotDescriptor's it is important that they are
+ // inserted into the correct 'TupleDescriptor'. This stack is used to keep track of the
+ // current (most nested) 'TupleDescriptor' at each step.
+ //
+ // When a new top-level path is registered, the root descriptor of the path is pushed on
+ // the stack. When registering a complex type, we push its item tuple desc on the stack
+ // before analysing the children, and pop it afterwards. New 'SlotDescriptor's are
+ // always added to the 'TupleDescriptor' at the top of the stack.
+ //
+ // To ensure that every push operation has a corresponding pop operation, these should
+ // not be called manually. Instead, use 'TupleStackGuard' objects in try-with-resources
+ // blocks; see more in its documentation.
+ private Deque<TupleDescriptor> tupleStack_ = new ArrayDeque<>();
+
+ // When created, pushes the provided TupleDescriptor to the enclosing Analyzer object's
+ // 'tupleStack_' and pops it in the close() method. Implements the AutoCloseable
+ // interface so it can be used in try-with-resources blocks.
+ private class TupleStackGuard implements AutoCloseable {
+ private final TupleDescriptor tupleDesc_;
+ private boolean isClosed_ = false;
+ private final int stackLen_;
+
+ public TupleStackGuard(TupleDescriptor tupleDesc) {
+ Preconditions.checkNotNull(tupleStack_);
+ Preconditions.checkNotNull(tupleDesc);
+ tupleDesc_ = tupleDesc;
+ stackLen_ = tupleStack_.size();
+ tupleStack_.push(tupleDesc_);
+ }
+
+ @Override
+ public void close() {
+ if (!isClosed_) {
+ Preconditions.checkState(tupleStack_.peek() == tupleDesc_);
+ tupleStack_.pop();
+ Preconditions.checkState(tupleStack_.size() == stackLen_);
+ isClosed_ = true;
+ }
+ }
+ }
+
// Required Operation type: Read, write, any(read or write).
public enum OperationType {
READ,
@@ -1496,55 +1540,74 @@ public class Analyzer {
/**
* Returns an existing or new SlotDescriptor for the given path.
- * If duplicateIfCollections is true, then always returns
- * a new empty SlotDescriptor for paths with a collection-typed destination.
+ * If 'duplicateIfCollections' is true, then always returns a new empty SlotDescriptor
+ * for paths with a collection-typed destination.
*/
- public SlotDescriptor registerSlotRef(Path slotPath, boolean duplicateIfCollections)
- throws AnalysisException {
+ public SlotDescriptor registerSlotRef(Path slotPath,
+ boolean duplicateIfCollections) throws AnalysisException {
Preconditions.checkState(slotPath.isRootedAtTuple());
- if (slotPath.destType().isCollectionType() && duplicateIfCollections) {
- // Register a new slot descriptor for collection types. The BE currently
- // relies on this behavior for setting unnested collection slots to NULL.
- return createAndRegisterSlotDesc(slotPath, false);
- }
- // SlotRefs with scalar or struct types are registered against the slot's
- // fully-qualified lowercase path.
- List<String> key = slotPath.getFullyQualifiedRawPath();
- Preconditions.checkState(key.stream().allMatch(s -> s.equals(s.toLowerCase())),
- "Slot paths should be lower case: " + key);
- SlotDescriptor existingSlotDesc = slotPathMap_.get(key);
- if (existingSlotDesc != null) return existingSlotDesc;
+ // If 'tupleStack_' is empty then this is a top level call to this function (not a
+ // recursive call) and we push the root TupleDescriptor to 'tupleStack_'.
+ try (TupleStackGuard guard = tupleStack_.isEmpty()
+ ? new TupleStackGuard(slotPath.getRootDesc()) : null) {
+ if (slotPath.destType().isCollectionType() && duplicateIfCollections) {
+ // Register a new slot descriptor for collection types. The BE currently
+ // relies on this behavior for setting unnested collection slots to NULL.
+ return createAndRegisterRawSlotDesc(slotPath, false);
+ }
+ // SlotRefs with scalar or struct types are registered against the slot's
+ // fully-qualified lowercase path.
+ List<String> key = slotPath.getFullyQualifiedRawPath();
+ Preconditions.checkState(key.stream().allMatch(s -> s.equals(s.toLowerCase())),
+ "Slot paths should be lower case: " + key);
+ SlotDescriptor existingSlotDesc = slotPathMap_.get(key);
+ if (existingSlotDesc != null) return existingSlotDesc;
+
+ return createAndRegisterSlotDesc(slotPath);
+ }
+ }
+ private SlotDescriptor createAndRegisterSlotDesc(Path slotPath)
+ throws AnalysisException {
if (slotPath.destType().isCollectionType()) {
SlotDescriptor result = registerCollectionSlotRef(slotPath);
- result.setPath(slotPath);
- slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), result);
- registerColumnPrivReq(result);
+ registerSlotDesc(slotPath, result, true);
return result;
}
+ SlotDescriptor result = createAndRegisterRawSlotDesc(slotPath, true);
if (slotPath.destType().isStructType()) {
- // 'slotPath' refers to a struct that has no ancestor that is also needed in the
- // query. We also create the child slot descriptors.
- SlotDescriptor result = createAndRegisterSlotDesc(slotPath, true);
createStructTuplesAndSlotDescs(result);
- return result;
}
+ return result;
+ }
- SlotDescriptor result = createAndRegisterSlotDesc(slotPath, true);
+ /**
+ * Creates a new SlotDescriptor with path 'slotPath' in the tuple at the top of
+ * 'tupleStack_'. Does not create SlotDescriptors for children of complex types. If
+ * 'insertIntoSlotPath' is true, inserts the new SlotDescriptor into 'slotPathMap_'.
+ * Also registers a column-level privilege request for the new SlotDescriptor.
+ */
+ private SlotDescriptor createAndRegisterRawSlotDesc(Path slotPath,
+ boolean insertIntoSlotPathMap) {
+ final SlotDescriptor result = addSlotDescriptorAtCurrentLevel();
+ registerSlotDesc(slotPath, result, insertIntoSlotPathMap);
return result;
}
- private SlotDescriptor createAndRegisterSlotDesc(Path slotPath,
+ /**
+ * Sets 'slotPath' as the path of 'desc' and registers a column-level privilege request
+ * for it. If 'insertIntoSlotPath' is true, inserts the new SlotDescriptor into
+ * 'slotPathMap_'.
+ */
+ private void registerSlotDesc(Path slotPath, SlotDescriptor desc,
boolean insertIntoSlotPathMap) {
Preconditions.checkState(slotPath.isRootedAtTuple());
- final SlotDescriptor result = addSlotDescriptor(slotPath.getRootDesc());
- result.setPath(slotPath);
+ desc.setPath(slotPath);
if (insertIntoSlotPathMap) {
- slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), result);
+ slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), desc);
}
- registerColumnPrivReq(result);
- return result;
+ registerColumnPrivReq(desc);
}
public void createStructTuplesAndSlotDescs(SlotDescriptor desc)
@@ -1557,26 +1620,38 @@ public class Analyzer {
structTuple.setType(type);
structTuple.setParentSlotDesc(desc);
desc.setItemTupleDesc(structTuple);
- for (StructField structField : type.getFields()) {
- SlotDescriptor childDesc = getDescTbl().addSlotDescriptor(structTuple);
- globalState_.blockBySlot.put(childDesc.getId(), this);
- // 'slotPath' could be null e.g. when the query has an order by clause and
- // this is the sorting tuple.
- if (slotPath != null) {
- Path relPath = Path.createRelPath(slotPath, structField.getName());
- relPath.resolve();
- childDesc.setPath(relPath);
- registerColumnPrivReq(childDesc);
- slotPathMap_.putIfAbsent(relPath.getFullyQualifiedRawPath(), childDesc);
- }
- childDesc.setType(structField.getType());
- if (childDesc.getType().isStructType()) {
- createStructTuplesAndSlotDescs(childDesc);
+ try (TupleStackGuard guard = new TupleStackGuard(structTuple)) {
+ for (StructField structField : type.getFields()) {
+ // 'slotPath' could be null e.g. when the query has an order by clause and
+ // this is the sorting tuple.
+ // TODO: IMPALA-10939: When we enable collections in sorting tuples we need to
+ // revisit this. Currently collection SlotDescriptors cannot be created without a
+ // path.
+ if (slotPath == null) {
+ createStructTuplesAndSlotDescsWithoutPath(slotPath, structField);
+ } else {
+ Path childRelPath = Path.createRelPath(slotPath, structField.getName());
+ childRelPath.resolve();
+ SlotDescriptor childDesc = createAndRegisterSlotDesc(childRelPath);
+ }
}
}
}
+ private void createStructTuplesAndSlotDescsWithoutPath(Path slotPath,
+ StructField structField) throws AnalysisException {
+ Preconditions.checkState(slotPath == null);
+ Preconditions.checkState(!structField.getType().isCollectionType());
+
+ SlotDescriptor childDesc = addSlotDescriptorAtCurrentLevel();
+ childDesc.setType(structField.getType());
+
+ if (childDesc.getType().isStructType()) {
+ createStructTuplesAndSlotDescs(childDesc);
+ }
+ }
+
/**
* Registers a collection and its descendants.
* Creates a CollectionTableRef for all collections on the path.
@@ -1604,18 +1679,20 @@ public class Analyzer {
SlotDescriptor desc = ((SlotRef) collTblRef.getCollectionExpr()).getDesc();
desc.setIsMaterializedRecursively(true);
- if (slotPath.destType().isArrayType()) {
- // Resolve path
- List<String> rawPathToItem = Arrays.asList(Path.ARRAY_ITEM_FIELD_NAME);
- resolveAndRegisterDescendantPath(collTblRef, rawPathToItem);
- } else {
- Preconditions.checkState(slotPath.destType().isMapType());
+ try (TupleStackGuard guard = new TupleStackGuard(desc.getItemTupleDesc())) {
+ if (slotPath.destType().isArrayType()) {
+ // Resolve path
+ List<String> rawPathToItem = Arrays.asList(Path.ARRAY_ITEM_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToItem);
+ } else {
+ Preconditions.checkState(slotPath.destType().isMapType());
- List<String> rawPathToKey = Arrays.asList(Path.MAP_KEY_FIELD_NAME);
- resolveAndRegisterDescendantPath(collTblRef, rawPathToKey);
+ List<String> rawPathToKey = Arrays.asList(Path.MAP_KEY_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToKey);
- List<String> rawPathToValue = Arrays.asList(Path.MAP_VALUE_FIELD_NAME);
- resolveAndRegisterDescendantPath(collTblRef, rawPathToValue);
+ List<String> rawPathToValue = Arrays.asList(Path.MAP_VALUE_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToValue);
+ }
}
return desc;
@@ -1627,10 +1704,6 @@ public class Analyzer {
boolean isResolved = resolvedPath.resolve();
Preconditions.checkState(isResolved);
- if (resolvedPath.destType().isStructType()) {
- throw new AnalysisException(
- "STRUCT type inside collection types is not supported.");
- }
if (resolvedPath.destType().isBinary()) {
throw new AnalysisException(
"Binary type inside collection types is not supported (IMPALA-11491).");
@@ -1698,6 +1771,16 @@ public class Analyzer {
return result;
}
+ /**
+ * Creates a new slot descriptor and related state in globalState. The new slot
+ * descriptor will be created in the tuple at the top of 'tupleStack_'.
+ */
+ private SlotDescriptor addSlotDescriptorAtCurrentLevel() {
+ TupleDescriptor tupleDesc = tupleStack_.peek();
+ Preconditions.checkNotNull(tupleDesc);
+ return addSlotDescriptor(tupleDesc);
+ }
+
/**
* Adds a new slot descriptor in tupleDesc that is identical to srcSlotDesc
* except for the path and slot id.
diff --git a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
index 2656886d5..1199bd086 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
@@ -19,6 +19,7 @@ package org.apache.impala.analysis;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.FeView;
+import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
import com.google.common.base.Preconditions;
@@ -99,6 +100,13 @@ public class CollectionTableRef extends TableRef {
if (resolvedPath_.getRootDesc() != null) {
sourceView = resolvedPath_.getRootDesc().getSourceView();
}
+ if (zippingUnnestType_ == ZippingUnnestType.FROM_CLAUSE_ZIPPING_UNNEST) {
+ UnnestExpr.verifyNotInsideStruct(resolvedPath_);
+
+ Type type = resolvedPath_.getMatchedTypes().get(
+ resolvedPath_.getMatchedTypes().size() - 1);
+ UnnestExpr.verifyContainsNoStruct(type);
+ }
if (sourceView != null && zippingUnnestType_ ==
ZippingUnnestType.FROM_CLAUSE_ZIPPING_UNNEST) {
String implicitAlias = rawPath_.get(rawPath_.size() - 1).toLowerCase();
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index 38c9508b5..2a57e6ec8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -297,137 +297,152 @@ public class InlineViewRef extends TableRef {
SlotDescriptor slotDesc = analyzer.registerSlotRef(p, false);
slotDesc.setSourceExpr(colExpr);
slotDesc.setStats(ColumnStats.fromExpr(colExpr));
- SlotRef slotRef = new SlotRef(slotDesc);
- smap_.put(slotRef, colExpr);
- baseTblSmap_.put(slotRef, baseTableExpr);
+
+ putExprsIntoSmaps(analyzer, slotDesc, colExpr, baseTableExpr);
+ }
+
+ // Inserts elements into 'smap_' and 'baseTblSmap_'. The key will be a new slot ref
+ // created from 'slotDesc' in both smaps; the value will be 'colExpr' in 'smap_' and
+ // 'baseTableExpr' in 'baseTblSmap_'. If 'recurse' is true, also adds struct members and
+ // collection items.
+ private void putExprsIntoSmaps(Analyzer analyzer,
+ SlotDescriptor slotDesc, Expr colExpr, Expr baseTableExpr, boolean recurse) {
+ SlotRef key = new SlotRef(slotDesc);
+
+ smap_.put(key, colExpr);
+ baseTblSmap_.put(key, baseTableExpr);
+
if (createAuxPredicate(colExpr)) {
analyzer.createAuxEqPredicate(new SlotRef(slotDesc), colExpr.clone());
}
- if (colExpr.getType().isCollectionType()) {
- // Calling registerSlotRef() above created a new SlotDescriptor + TupleDescriptor
- // hierarchy for Array types. Walk through this hiararchy and add all slot refs
- // to smap_ and baseTblSmap_.
- // Source must be a SlotRef
- SlotRef srcSlotRef = (SlotRef) colExpr;
- SlotRef baseTableSlotRef = (SlotRef) baseTableExpr;
- SlotDescriptor srcSlotDesc = srcSlotRef.getDesc();
- SlotDescriptor baseTableSlotDesc = baseTableSlotRef.getDesc();
- TupleDescriptor itemTupleDesc = slotDesc.getItemTupleDesc();
- TupleDescriptor srcItemTupleDesc = srcSlotDesc.getItemTupleDesc();
- TupleDescriptor baseTableItemTupleDesc = baseTableSlotDesc.getItemTupleDesc();
- // We don't recurse deeper and only add the immediate item child to the
- // substitution map. This is enough both for collections in select list and in
- // from clause.
- if (itemTupleDesc != null) {
- Preconditions.checkState(srcItemTupleDesc != null);
- Preconditions.checkState(baseTableItemTupleDesc != null);
-
- final int num_slots = itemTupleDesc.getSlots().size();
- Preconditions.checkState(srcItemTupleDesc.getSlots().size() == num_slots);
- Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() == num_slots);
-
- // There is one slot for arrays and two for maps. When we add support for structs
- // in collections in the select list, there may be more slots.
- for (int i = 0; i < num_slots; i++) {
- SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(i);
- SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(i);
- SlotDescriptor baseTableItemSlotDesc = baseTableItemTupleDesc.getSlots().get(i);
- SlotRef itemSlotRef = new SlotRef(itemSlotDesc);
- SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc);
- SlotRef beseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc);
- smap_.put(itemSlotRef, srcItemSlotRef);
- baseTblSmap_.put(itemSlotRef, beseTableItemSlotRef);
- if (createAuxPredicate(colExpr)) {
- analyzer.createAuxEqPredicate(
- new SlotRef(itemSlotDesc), srcItemSlotRef.clone());
- }
- }
- }
- }
+ if (recurse) {
+ if (colExpr.getType().isCollectionType()) {
+ Preconditions.checkState(colExpr instanceof SlotRef);
+ Preconditions.checkState(baseTableExpr instanceof SlotRef);
- if (colExpr.getType().isStructType()) {
- Preconditions.checkState(colExpr instanceof SlotRef);
- Preconditions.checkState(baseTableExpr instanceof SlotRef);
- putStructMembersIntoSmap(smap_, slotDesc, (SlotRef) colExpr);
- putStructMembersIntoSmap(baseTblSmap_, slotDesc, (SlotRef) baseTableExpr);
- createAuxPredicatesForStructMembers(analyzer, slotDesc, (SlotRef) colExpr);
- }
- }
+ putCollectionItemsIntoSmaps(analyzer, slotDesc, (SlotRef) colExpr,
+ (SlotRef) baseTableExpr);
+ } else if (colExpr.getType().isStructType()) {
+ Preconditions.checkState(colExpr instanceof SlotRef);
+ Preconditions.checkState(baseTableExpr instanceof SlotRef);
- // Puts the fields of 'rhsStruct' into 'smap' as right hand side (mapped) values,
- // recursively. The keys (left hand side values) are constructed based on the
- // corresponding slot descriptors in the itemTupleDesc of 'lhsSlotDesc'.
- private void putStructMembersIntoSmap(ExprSubstitutionMap smap,
- SlotDescriptor lhsSlotDesc, SlotRef rhsStruct) {
- for (Pair<SlotDescriptor, SlotRef> pair :
- getStructSlotDescSlotRefPairs(lhsSlotDesc, rhsStruct)) {
- SlotDescriptor lhs = pair.first;
- SlotRef rhs = pair.second;
- smap.put(new SlotRef(lhs), rhs);
+ putStructMembersIntoSmaps(analyzer, slotDesc, (SlotRef) colExpr,
+ (SlotRef) baseTableExpr);
+ }
}
}
- private void createAuxPredicatesForStructMembers(Analyzer analyzer,
- SlotDescriptor structSlotDesc, SlotRef structExpr) {
- for (Pair<SlotDescriptor, SlotRef> pair :
- getStructSlotDescSlotRefPairs(structSlotDesc, structExpr)) {
- SlotDescriptor structMemberSlotDesc = pair.first;
- SlotRef structMemberExpr = pair.second;
+ private void putExprsIntoSmaps(Analyzer analyzer,
+ SlotDescriptor slotDesc, Expr colExpr, Expr baseTableExpr) {
+ putExprsIntoSmaps(analyzer, slotDesc, colExpr, baseTableExpr, true);
+ }
- if (createAuxPredicate(structMemberExpr)) {
- analyzer.createAuxEqPredicate(
- new SlotRef(structMemberSlotDesc), structMemberExpr.clone());
+ // Add slot refs for collection items to smap_ and baseTblSmap_.
+ private void putCollectionItemsIntoSmaps(Analyzer analyzer, SlotDescriptor slotDesc,
+ SlotRef colExpr, SlotRef baseTableExpr) {
+ // Source must be a SlotRef
+ SlotDescriptor srcSlotDesc = colExpr.getDesc();
+ SlotDescriptor baseTableSlotDesc = baseTableExpr.getDesc();
+
+ TupleDescriptor itemTupleDesc = slotDesc.getItemTupleDesc();
+ TupleDescriptor srcItemTupleDesc = srcSlotDesc.getItemTupleDesc();
+ TupleDescriptor baseTableItemTupleDesc = baseTableSlotDesc.getItemTupleDesc();
+ if (itemTupleDesc != null) {
+ Preconditions.checkState(srcItemTupleDesc != null);
+ Preconditions.checkState(baseTableItemTupleDesc != null);
+
+ final int num_slots = itemTupleDesc.getSlots().size();
+ // There is one slot for arrays and two for maps.
+ Preconditions.checkState(num_slots == 1 || num_slots == 2);
+ Preconditions.checkState(srcItemTupleDesc.getSlots().size() == num_slots);
+ Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() == num_slots);
+
+ for (int i = 0; i < num_slots; i++) {
+ SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(i);
+ SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(i);
+ SlotDescriptor baseTableItemSlotDesc = baseTableItemTupleDesc.getSlots().get(i);
+ SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc);
+ SlotRef baseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc);
+
+ Preconditions.checkState(itemSlotDesc.getType().equals(srcItemSlotRef.getType()));
+ Preconditions.checkState(itemSlotDesc.getType().equals(
+ baseTableItemSlotRef.getType()));
+
+ // We don't recurse deeper and only add the immediate item child to the
+ // substitution map. This is enough both for collections in select list and in
+ // from clause.
+ putExprsIntoSmaps(analyzer, itemSlotDesc, srcItemSlotRef, baseTableItemSlotRef,
+ false);
}
}
}
- /**
- * Given a slot desc and a SlotRef expression, both referring to the same struct,
- * returns a list of corresponding SlotDescriptor/SlotRef pairs of the struct members,
- * recursively.
- */
- private List<Pair<SlotDescriptor, SlotRef>> getStructSlotDescSlotRefPairs(
- SlotDescriptor structSlotDesc, SlotRef structExpr) {
- Preconditions.checkState(structSlotDesc.getType().isStructType());
- Preconditions.checkState(structExpr.getType().isStructType());
- Preconditions.checkState(structSlotDesc.getType().equals(structExpr.getType()));
+ // Put struct members into 'smap_' and 'baseTblSmap_'. 'slotDesc', 'colExpr' and
+ // 'baseTableExpr' should all belong to the same struct. The struct tree is traversed;
+ // keys in both maps will be slot refs created from the elements of the tree rooted at
+ // 'slotDesc'. The values in 'smap_' will be the expressions in the tree of 'colExpr'
+ // and the values in 'baseTblSmap_' will be the expressions in the tree of
+ // 'baseTableExpr'
+ private void putStructMembersIntoSmaps(Analyzer analyzer, SlotDescriptor slotDesc,
+ SlotRef colExpr, SlotRef baseTableExpr) {
+ Preconditions.checkState(slotDesc.getType().isStructType());
+ Preconditions.checkState(colExpr.getType().isStructType());
+ Preconditions.checkState(baseTableExpr.getType().isStructType());
- List<Pair<SlotDescriptor, SlotRef>> result = new ArrayList<>();
+ Preconditions.checkState(slotDesc.getType().equals(colExpr.getType()));
+ Preconditions.checkState(slotDesc.getType().equals(baseTableExpr.getType()));
- TupleDescriptor lhsItemTupleDesc = structSlotDesc.getItemTupleDesc();
- Preconditions.checkNotNull(lhsItemTupleDesc);
- List<SlotDescriptor> lhsChildSlotDescs = lhsItemTupleDesc.getSlots();
- Preconditions.checkState(
- lhsChildSlotDescs.size() == structExpr.getChildren().size());
- for (int i = 0; i < lhsChildSlotDescs.size(); i++) {
- SlotDescriptor lhsChildSlotDesc = lhsChildSlotDescs.get(i);
+ TupleDescriptor itemTupleDesc = slotDesc.getItemTupleDesc();
+ Preconditions.checkNotNull(itemTupleDesc);
- Expr rhsChild = structExpr.getChildren().get(i);
- Preconditions.checkState(rhsChild instanceof SlotRef);
- SlotRef rhsChildSlotRef = (SlotRef) rhsChild;
+ List<SlotDescriptor> childSlotDescs = itemTupleDesc.getSlots();
+ Preconditions.checkState(childSlotDescs.size() == colExpr.getChildren().size());
+ Preconditions.checkState(childSlotDescs.size() == baseTableExpr.getChildren().size());
- List<String> lhsRawPath = lhsChildSlotDesc.getPath().getRawPath();
- Path rhsPath = rhsChildSlotRef.getResolvedPath();
+ for (int i = 0; i < childSlotDescs.size(); i++) {
+ SlotDescriptor childSlotDesc = childSlotDescs.get(i);
- // The path can be null in the case of the sorting tuple.
- if (rhsPath != null) {
- List<String> rhsRawPath = rhsPath.getRawPath();
+ Expr childColExpr = colExpr.getChildren().get(i);
+ Preconditions.checkState(childColExpr instanceof SlotRef);
+ SlotRef childColExprSlotRef = (SlotRef) childColExpr;
- // Check that the children come in the same order on both lhs and rhs. If not, the
- // last part of the paths would be different.
- Preconditions.checkState(lhsRawPath.get(lhsRawPath.size() - 1)
- .equals(rhsRawPath.get(rhsRawPath.size() - 1)));
+ Expr childBaseTableExpr = baseTableExpr.getChildren().get(i);
+ Preconditions.checkState(childBaseTableExpr instanceof SlotRef);
+ SlotRef childBaseTableExprSlotRef = (SlotRef) childBaseTableExpr;
- result.add(new Pair(lhsChildSlotDesc, rhsChildSlotRef));
+ Path childColExprPath = childColExprSlotRef.getResolvedPath();
+ Path childBaseTableExprPath = childBaseTableExprSlotRef.getResolvedPath();
- if (rhsChildSlotRef.getType().isStructType()) {
- result.addAll(
- getStructSlotDescSlotRefPairs(lhsChildSlotDesc, rhsChildSlotRef));
- }
+ // The path can be null in the case of the sorting tuple.
+ if (childColExprPath != null) {
+ verifySameChild(childSlotDesc.getPath(), childColExprPath,
+ childBaseTableExprPath);
+
+ putExprsIntoSmaps(analyzer, childSlotDesc, childColExprSlotRef,
+ childBaseTableExprSlotRef);
}
}
- return result;
+ }
+
+ // Verify that the paths belong to the same struct child.
+ private static void verifySameChild(Path childSlotDescPath, Path childColExprPath,
+ Path childBaseTableExprPath) {
+ List<String> childSlotDescRawPath = childSlotDescPath.getRawPath();
+ List<String> childColExprRawPath = childColExprPath.getRawPath();
+ List<String> childBaseTableExprRawPath = childBaseTableExprPath.getRawPath();
+
+ // Check that the children come in the same order for all of 'slotDesc', 'colExpr'
+ // and 'baseTableExpr'. If not, the last part of the paths would be different.
+ String childSlotDescPathEnd =
+ childSlotDescRawPath.get(childSlotDescRawPath.size() - 1);
+ String childColExprPathEnd =
+ childColExprRawPath.get(childColExprRawPath.size() - 1);
+ String childBaseTableExprPathEnd =
+ childBaseTableExprRawPath.get(childBaseTableExprRawPath.size() - 1);
+
+ Preconditions.checkState(childSlotDescPathEnd.equals(childColExprPathEnd));
+ Preconditions.checkState(childSlotDescPathEnd.equals(childBaseTableExprPathEnd));
}
/**
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
index 898059842..7ca2cf958 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
@@ -217,7 +217,8 @@ public class SlotDescriptor {
}
/**
- * Returns the slot descs of the structs that contain this slot desc, recursively.
+ * Returns the slot descs of the structs that contain this slot desc, recursively; stops
+ * at collection items, does not continue to the parent collection.
* For an example struct 'outer: <middle: <inner: <i: int>>>', called for the slot desc
* of 'i', the returned list will contain the slot descs of 'inner', 'middle' and
* 'outer'.
@@ -229,7 +230,8 @@ public class SlotDescriptor {
}
/**
- * Returns the tuple descs enclosing this slot desc, recursively.
+ * Returns the tuple descs enclosing this slot desc, recursively; stops at collection
+ * items, does not continue to the parent collection.
* For an example struct 'outer: <middle: <inner: <i: int>>>', called for the slot desc
* of 'i', the returned list will contain the 'itemTupleDesc_'s of 'inner', 'middle'
* and 'outer' as well as the tuple desc of the main tuple (the 'parent_' of the slot
@@ -241,6 +243,26 @@ public class SlotDescriptor {
return result;
}
+ /**
+ * Climbs up the struct hierarchy and returns the tuple desc that holds the top level
+ * struct that contains this slot desc; stops at collection items, does not continue to
+ * the parent collection.
+ * For a slot desc that is not within a struct, simply returns 'parent_'.
+ * For an example struct 'outer: <middle: <inner: <i: int>>>', called for the slot desc
+ * of 'i', returns the tuple desc of the main tuple (the 'parent_' of the slot desc of
+ * 'outer').
+ */
+ public TupleDescriptor getTopEnclosingTupleDesc() {
+ List<TupleDescriptor> enclosingTuples = getEnclosingTupleDescs();
+ if (enclosingTuples.isEmpty()) {
+ Preconditions.checkState(getParent() == null);
+ return null;
+ }
+
+ // Return the last enclosing tuple, going upward.
+ return enclosingTuples.get(enclosingTuples.size() - 1);
+ }
+
/**
* Returns the size of the slot without null indicators.
*
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index e9fa5d5ed..8f64b52c6 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -212,10 +212,6 @@ public class SlotRef extends Expr {
throw new AnalysisException("Unsupported type '"
+ fieldType.toSql() + "' in '" + toSql() + "'.");
}
- if (fieldType.isCollectionType()) {
- throw new AnalysisException("Struct containing a collection type is not " +
- "allowed in the select list.");
- }
if (fieldType.isBinary()) {
throw new AnalysisException("Struct containing a BINARY type is not " +
"allowed in the select list (IMPALA-11491).");
diff --git a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
index d5c42653f..04f522e5a 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
@@ -294,7 +294,7 @@ public class TupleDescriptor {
* Materialize all slots.
*/
public void materializeSlots() {
- for (SlotDescriptor slot: slots_) slot.setIsMaterialized(true);
+ for (SlotDescriptor slot: getSlotsRecursively()) slot.setIsMaterialized(true);
}
public TTupleDescriptor toThrift(Integer tableId) {
diff --git a/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java b/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java
index ec3355077..932a4fce6 100644
--- a/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/UnnestExpr.java
@@ -20,6 +20,8 @@ package org.apache.impala.analysis;
import org.apache.impala.analysis.Path.PathType;
import org.apache.impala.analysis.TableRef.ZippingUnnestType;
import org.apache.impala.catalog.TableLoadingException;
+import org.apache.impala.catalog.ArrayType;
+import org.apache.impala.catalog.MapType;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
@@ -62,6 +64,12 @@ public class UnnestExpr extends SlotRef {
// find the corresponding CollectionTableRef during resolution.
Path resolvedPath = resolveAndVerifyRawPath(analyzer);
Preconditions.checkNotNull(resolvedPath);
+ verifyNotInsideStruct(resolvedPath);
+
+ Type type = resolvedPath.getMatchedTypes().get(
+ resolvedPath.getMatchedTypes().size() - 1);
+ verifyContainsNoStruct(type);
+
if (!rawPathWithoutItem_.isEmpty()) rawPathWithoutItem_.clear();
rawPathWithoutItem_.addAll(rawPath_);
@@ -83,6 +91,37 @@ public class UnnestExpr extends SlotRef {
analyzer.addZippingUnnestTupleId(desc_.getParent().getId());
}
+ // Verifies that the given path does not refer to a value that is within a struct.
+ // Note: If using the FROM clause zipping unnest syntax, this check has to be performed
+ // in CollectionTableRef.analyze().
+ public static void verifyNotInsideStruct(Path resolvedPath) throws AnalysisException {
+ for (Type type : resolvedPath.getMatchedTypes()) {
+ if (type.isStructType()) {
+ throw new AnalysisException(
+ "Zipping unnest on an array that is within a struct is not supported.");
+ }
+ }
+ }
+
+ // Verifies that the given type does not contain a struct. Descends along array items
+ // and map keys/values.
+ // Note: If using the FROM clause zipping unnest syntax, this check has to be performed
+ // in CollectionTableRef.analyze().
+ public static void verifyContainsNoStruct(Type type) throws AnalysisException {
+ if (type.isStructType()) {
+ throw new AnalysisException(
+ "Zipping unnest on an array that (recursively) " +
+ "contains a struct is not supported.");
+ } else if (type.isArrayType()) {
+ ArrayType arrType = (ArrayType) type;
+ verifyContainsNoStruct(arrType.getItemType());
+ } else if (type.isMapType()) {
+ MapType mapType = (MapType) type;
+ verifyContainsNoStruct(mapType.getKeyType());
+ verifyContainsNoStruct(mapType.getValueType());
+ }
+ }
+
private void verifyTableRefs(Analyzer analyzer) throws AnalysisException {
for (TableRef ref : analyzer.getTableRefs().values()) {
if (ref instanceof CollectionTableRef) {
@@ -210,4 +249,4 @@ public class UnnestExpr extends SlotRef {
}
return super.isBoundByTupleIds(tids);
}
-}
\ No newline at end of file
+}
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 7939e72c2..197f8731f 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -863,7 +863,12 @@ public class SingleNodePlanner {
SlotRef collectionExpr =
(SlotRef) collectionTableRef.getCollectionExpr();
if (collectionExpr != null) {
- requiredTids.add(collectionExpr.getDesc().getParent().getId());
+ // If the collection is within a (possibly nested) struct, add the tuple in
+ // which the top level struct is located.
+ SlotDescriptor desc = collectionExpr.getDesc();
+ List<TupleDescriptor> enclosingTupleDescs = desc.getEnclosingTupleDescs();
+ TupleDescriptor topTuple = desc.getTopEnclosingTupleDesc();
+ requiredTids.add(topTuple.getId());
} else {
requiredTids.add(collectionTableRef.getResolvedPath().getRootDesc().getId());
}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index ebad509d1..6e402a6a5 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -632,10 +632,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
testTableRefPath("select 1 from d.t7.c3.item.a2.item.a3", path(2, 0, 1, 0, 2), null);
testSlotRefPath("select item from d.t7.c3.a2.a3", path(2, 0, 1, 0, 2, 0));
testSlotRefPath("select item from d.t7.c3.item.a2.item.a3", path(2, 0, 1, 0, 2, 0));
- AnalysisContext ctx = createAnalysisCtx();
- ctx.getQueryOptions().setDisable_codegen(true);
- AnalysisError("select item from d.t7.c3", ctx,
- "Struct containing a collection type is not allowed in the select list.");
+ testSlotRefPath("select item from d.t7.c3", path(2, 0));
// Test path assembly with multiple tuple descriptors.
testTableRefPath("select 1 from d.t7, t7.c3, c3.a2, a2.a3",
path(2, 0, 1, 0, 2), path(2, 0, 1, 0, 2));
@@ -784,17 +781,16 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
"Querying STRUCT is only supported for ORC and Parquet file formats.");
AnalyzesOk("select alltypes from functional_orc_def.complextypes_structs", ctx);
- // Check if a struct in the select list raises an error if it contains collections.
+ // Check that a struct in the select list doesn't raise an error if it contains
+ // collections.
addTestTable(
"create table nested_structs (s1 struct<s2:struct<i:int>>) stored as orc");
addTestTable("create table nested_structs_with_list " +
"(s1 struct<s2:struct<a:array<int>>>) stored as orc");
AnalyzesOk("select s1 from nested_structs", ctx);
AnalyzesOk("select s1.s2 from nested_structs", ctx);
- AnalysisError("select s1 from nested_structs_with_list", ctx, "Struct containing " +
- "a collection type is not allowed in the select list.");
- AnalysisError("select s1.s2 from nested_structs_with_list", ctx, "Struct " +
- "containing a collection type is not allowed in the select list.");
+ AnalyzesOk("select s1 from nested_structs_with_list", ctx);
+ AnalyzesOk("select s1.s2 from nested_structs_with_list", ctx);
}
@Test
@@ -1088,19 +1084,15 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalyzesOk("select * from (select int_map, int_map_array from " +
"functional_parquet.complextypestbl) v",ctx);
- ctx.getQueryOptions().setDisable_codegen(false);
-
AnalyzesOk("select * from functional_parquet.complextypes_arrays",ctx);
AnalyzesOk("select * from " +
"functional_parquet.complextypes_arrays_only_view",ctx);
AnalyzesOk("select v.id, v.* from " +
"(select * from functional_parquet.complextypes_arrays) v",ctx);
- AnalysisError("select * from functional.allcomplextypes",
- ctx,"STRUCT type inside collection types is not supported.");
-
- AnalysisError("select * from functional_orc_def.complextypestbl", ctx,
- "Struct containing a collection type is not allowed in the select list.");
+ // Allow also structs in collections and vice versa.
+ AnalyzesOk("select * from functional_parquet.allcomplextypes", ctx);
+ AnalyzesOk("select * from functional_orc_def.complextypestbl", ctx);
AnalysisError("select * from functional_parquet.binary_in_complex_types", ctx,
"Binary type inside collection types is not supported (IMPALA-11491).");
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 27b7f3cee..68d5453dd 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -3783,6 +3783,7 @@ map_char_key MAP<CHAR(3), INT>
map_varchar_key MAP<VARCHAR(3), STRING>
map_timestamp_key MAP<TIMESTAMP, STRING>
map_date_key MAP<DATE, STRING>
+struct_contains_map STRUCT<m: MAP<INT, STRING>, s: STRING>
---- DEPENDENT_LOAD_HIVE
INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
(1,
@@ -3799,13 +3800,141 @@ INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
map(cast("a" as VARCHAR(3)), "A", if(false, cast("" as VARCHAR(3)), NULL), NULL),
map(to_utc_timestamp("2022-12-10 08:15:12", "UTC"), "Saturday morning",
if(false, to_utc_timestamp("2022-12-10 08:15:12", "UTC"), NULL), "null"),
- map(to_date("2022-12-10"), "Saturday", if(false, to_date("2022-12-10"), NULL), "null")
+ map(to_date("2022-12-10"), "Saturday", if(false, to_date("2022-12-10"), NULL), "null"),
+ named_struct("m", map(1, "one", if(false, 1, NULL), "null"), "s", "some_string")
);
---- LOAD
====
---- DATASET
functional
---- BASE_TABLE_NAME
+collection_struct_mix
+---- COLUMNS
+id INT
+struct_contains_arr STRUCT<arr: ARRAY<INT>>
+struct_contains_map STRUCT<m: MAP<INT, STRING>>
+arr_contains_struct ARRAY<STRUCT<i: BIGINT>>
+arr_contains_nested_struct ARRAY<STRUCT<inner_struct: STRUCT<str: STRING, l: INT>, small: SMALLINT>>
+struct_contains_nested_arr STRUCT<arr: ARRAY<ARRAY<DATE>>, i: INT>
+all_mix MAP<INT, STRUCT<big: STRUCT<arr: ARRAY<STRUCT<inner_arr: ARRAY<ARRAY<INT>>, m: TIMESTAMP>>, n: INT>, small: STRUCT<str: STRING, i: INT>>>
+---- DEPENDENT_LOAD_HIVE
+INSERT OVERWRITE {db_name}{db_suffix}.{table_name} VALUES
+ (
+ 1,
+ named_struct("arr", array(1, 2, 3, 4, NULL, NULL, 5)),
+ named_struct("m", map(1, "one", 2, "two", 0, NULL)),
+ array(named_struct("i", 1L), named_struct("i", 2L), named_struct("i", 3L),
+ named_struct("i", 4L), NULL, named_struct("i", 5L), named_struct("i", NULL)),
+ array(named_struct("inner_struct", named_struct("str", "", "l", 0), "small", 2S), NULL,
+ named_struct("inner_struct", named_struct("str", "some_string", "l", 5), "small", 20S)),
+ named_struct("arr", array(array(to_date("2022-12-05"), to_date("2022-12-06"), NULL, to_date("2022-12-07")),
+ array(to_date("2022-12-08"), to_date("2022-12-09"), NULL)), "i", 2),
+ map(
+ 10,
+ named_struct(
+ "big", named_struct(
+ "arr", array(
+ named_struct(
+ "inner_arr", array(array(0, NULL, -1, -5, NULL, 8), array(20, NULL)),
+ "m", to_utc_timestamp("2022-12-05 14:30:00", "UTC")
+ ),
+ named_struct(
+ "inner_arr", array(array(12, 1024, NULL), array(NULL, NULL, 84), array(NULL, 15, NULL)),
+ "m", to_utc_timestamp("2022-12-06 16:20:52", "UTC")
+ )
+ ),
+ "n", 98
+ ),
+ "small", named_struct(
+ "str", "somestring",
+ "i", 100
+ )
+ )
+ )
+ ),
+ (
+ 2,
+ named_struct("arr", if(false, array(1), NULL)),
+ named_struct("m", if(false, map(1, "one"), NULL)),
+ array(named_struct("i", 100L), named_struct("i", 8L), named_struct("i", 35L),
+ named_struct("i", 45L), NULL, named_struct("i", 193L), named_struct("i", NULL)),
+ array(named_struct("inner_struct", if(false, named_struct("str", "", "l", 0), NULL), "small", 104S),
+ named_struct("inner_struct", named_struct("str", "aaa", "l", 28), "small", 105S), NULL),
+ named_struct("arr", array(array(to_date("2022-12-10"), to_date("2022-12-11"), NULL, to_date("2022-12-12")),
+ if(false, array(to_date("2022-12-12")), NULL)), "i", 2754),
+ map(
+ 20,
+ named_struct(
+ "big", named_struct(
+ "arr", array(
+ if(false, named_struct(
+ "inner_arr", array(array(0)),
+ "m", to_utc_timestamp("2022-12-10 08:01:05", "UTC")
+ ), NULL),
+ named_struct(
+ "inner_arr", array(array(12, 1024, NULL), array(NULL, NULL, 84), array(NULL, 15, NULL)),
+ "m", to_utc_timestamp("2022-12-10 08:15:12", "UTC")
+ )
+ ),
+ "n", 95
+ ),
+ "small", named_struct(
+ "str", "otherstring",
+ "i", 2048
+ )
+ ),
+ 21,
+ named_struct(
+ "big", named_struct(
+ "arr", if(false, array(
+ named_struct(
+ "inner_arr", array(array(0, NULL, -1, -5, NULL, 8), array(20, NULL)),
+ "m", to_utc_timestamp("2022-12-15 05:46:24", "UTC")
+ )
+ ), NULL),
+ "n", 8
+ ),
+ "small", named_struct(
+ "str", "textstring",
+ "i", 0
+ )
+ ),
+ 22,
+ named_struct(
+ "big", if(false, named_struct(
+ "arr", array(
+ named_struct(
+ "inner_arr", array(array(0)),
+ "m", if(false, to_utc_timestamp("2022-12-15 05:46:24", "UTC"), NULL)
+ )
+ ),
+ "n", 93
+ ), NULL),
+ "small", named_struct(
+ "str", "nextstring",
+ "i", 128
+ )
+ ),
+ 23,
+ NULL
+ )
+ );
+---- LOAD
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+collection_struct_mix_view
+---- CREATE
+SET disable_codegen=1;
+DROP VIEW IF EXISTS {db_name}{db_suffix}.{table_name};
+CREATE VIEW {db_name}{db_suffix}.{table_name}
+AS SELECT id, arr_contains_struct, arr_contains_nested_struct, struct_contains_nested_arr FROM {db_name}{db_suffix}.collection_struct_mix;
+---- LOAD
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
binary_tbl
---- COLUMNS
id INT
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index ad5abee48..6cf3b16a0 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -353,6 +353,12 @@ table_name:collection_tbl, constraint:restrict_to, table_format:orc/def/block
# In parquet we can't have NULL map keys but in ORC we can.
table_name:map_null_keys, constraint:restrict_to, table_format:orc/def/block
+table_name:collection_struct_mix, constraint:restrict_to, table_format:parquet/none/none
+table_name:collection_struct_mix, constraint:restrict_to, table_format:orc/def/block
+
+table_name:collection_struct_mix_view, constraint:restrict_to, table_format:parquet/none/none
+table_name:collection_struct_mix_view, constraint:restrict_to, table_format:orc/def/block
+
table_name:complextypes_maps_view, constraint:restrict_to, table_format:parquet/none/none
table_name:complextypes_maps_view, constraint:restrict_to, table_format:orc/def/block
diff --git a/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test b/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test
index 9d8cd8a89..184eb6b2a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/map_null_keys.test
@@ -15,12 +15,13 @@ select
map_char_key,
map_varchar_key,
map_timestamp_key,
- map_date_key
+ map_date_key,
+ struct_contains_map
from map_null_keys;
---- RESULTS
-1,'{true:"true",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1.75:"a",null:"null"}','{-1.75:"a",null:"null"}','{-1.8:"a",null:"null"}','{"one":1,null:null}','{"Mon":1,null:null}','{"a":"A",null:null}','{"2022-12-10 08:15:12":"Saturday morning",null:"null"}','{"2022-12-10":"Saturday",null:"null"}'
+1,'{true:"true",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1:"one",null:"null"}','{-1.75:"a",null:"null"}','{-1.75:"a",null:"null"}','{-1.8:"a",null:"null"}','{"one":1,null:null}','{"Mon":1,null:null}','{"a":"A",null:null}','{"2022-12-10 08:15:12":"Saturday morning",null:"null"}','{"2022-12-10":"Saturday",null:"null"}','{"m":{1:"one",null:"null"},"s":"some_string"}'
---- TYPES
-INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
+INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
=====
---- QUERY
-- Test that NULL map keys are printed correctly with STRINGIFY_MAP_KEYS=true.
@@ -39,10 +40,11 @@ select
map_char_key,
map_varchar_key,
map_timestamp_key,
- map_date_key
+ map_date_key,
+ struct_contains_map
from map_null_keys;
---- RESULTS
-1,'{"true":"true","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.8":"a","null":"null"}','{"one":1,"null":null}','{"Mon":1,"null":null}','{"a":"A","null":null}','{"2022-12-10 08:15:12":"Saturday morning","null":"null"}','{"2022-12-10":"Saturday","null":"null"}'
+1,'{"true":"true","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1":"one","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.75":"a","null":"null"}','{"-1.8":"a","null":"null"}','{"one":1,"null":null}','{"Mon":1,"null":null}','{"a":"A","null":null}','{"2022-12-10 08:15:12":"Saturday morning","null":"null"}','{"2022-12-10":"Saturday","null":"null"}','{"m":{"1":"one","null":"null"},"s":"some_string"}'
---- TYPES
-INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
+INT,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
=====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test
new file mode 100644
index 000000000..56ba1c5d1
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/mixed-collections-and-structs.test
@@ -0,0 +1,551 @@
+====
+---- QUERY
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select
+ id,
+ struct_contains_arr,
+ struct_contains_map,
+ arr_contains_struct,
+ arr_contains_nested_struct,
+ struct_contains_nested_arr,
+ all_mix
+from collection_struct_mix;
+---- RESULTS
+1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":" [...]
+2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"otherstring","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"str":"textstring","i":0}},22:{" [...]
+---- TYPES
+INT,STRING,STRING,STRING,STRING,STRING,STRING
+====
+---- QUERY
+select tbl.nested_struct from complextypestbl tbl;
+---- RESULTS
+'{"a":1,"b":[1],"c":{"d":[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]},"g":{"foo":{"h":{"i":[1.1]}}}}'
+'{"a":null,"b":[null],"c":{"d":[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]},"g":{"g1":{"h":{"i":[2.2,null]}},"g2":{"h":{"i":[]}},"g3":null,"g4":{"h":{"i":null}},"g5":{"h":null}}}'
+'{"a":null,"b":null,"c":{"d":[]},"g":{}}'
+'{"a":null,"b":null,"c":{"d":null},"g":null}'
+'{"a":null,"b":null,"c":null,"g":{"foo":{"h":{"i":[2.2,3.3]}}}}'
+'NULL'
+'{"a":7,"b":[2,3,null],"c":{"d":[[],[null],null]},"g":null}'
+'{"a":-1,"b":[-1],"c":{"d":[[{"e":-1,"f":"nonnullable"}]]},"g":{}}'
+---- TYPES
+STRING
+====
+---- QUERY
+select tbl.nested_struct.c from complextypestbl tbl;
+---- RESULTS
+'{"d":[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]}'
+'{"d":[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]}'
+'{"d":[]}'
+'{"d":null}'
+'NULL'
+'NULL'
+'{"d":[[],[null],null]}'
+'{"d":[[{"e":-1,"f":"nonnullable"}]]}'
+---- TYPES
+STRING
+====
+---- QUERY
+# Structs inside arrays are supported.
+select nested_struct.c.d from complextypestbl;
+---- RESULTS
+'[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]'
+'[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]'
+'[]'
+'NULL'
+'NULL'
+'NULL'
+'[[],[null],null]'
+'[[{"e":-1,"f":"nonnullable"}]]'
+---- TYPES
+STRING
+====
+---- QUERY
+# Structs inside maps are supported.
+select nested_struct.g from complextypestbl;
+---- RESULTS
+'{"foo":{"h":{"i":[1.1]}}}'
+'{"g1":{"h":{"i":[2.2,null]}},"g2":{"h":{"i":[]}},"g3":null,"g4":{"h":{"i":null}},"g5":{"h":null}}'
+'{}'
+'NULL'
+'{"foo":{"h":{"i":[2.2,3.3]}}}'
+'NULL'
+'NULL'
+'{}'
+---- TYPES
+STRING
+====
+---- QUERY
+# Select struct field from inline view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix)
+select sub.id, sub.struct_contains_nested_arr.arr from sub;
+---- RESULTS
+1,'[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]'
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select struct field from HMS view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, struct_contains_nested_arr.arr from collection_struct_mix_view;
+---- RESULTS
+1,'[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]'
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select array in struct from inline view and join-unnest it.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix)
+select id, arr.item from sub, sub.struct_contains_nested_arr.arr arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]'
+1,'["2022-12-08","2022-12-09",null]'
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select array in struct from HMS view and join-unnest it.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, arr.item from collection_struct_mix_view, collection_struct_mix_view.struct_contains_nested_arr.arr arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]'
+1,'["2022-12-08","2022-12-09",null]'
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Select array in struct from nested inline view and join-unnest it.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix),
+ sub2 as (select id, struct_contains_nested_arr s from sub)
+select id, item from sub2, sub2.s.arr arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]'
+1,'["2022-12-08","2022-12-09",null]'
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Unnest an array that contains structs.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, item from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}'
+1,'NULL'
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}'
+2,'{"inner_struct":null,"small":104}'
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Unnest an array that contains structs from a nested inline view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, arr_contains_nested_struct arr from collection_struct_mix),
+ sub2 as (select id, arr arr2 from sub)
+select id, item from sub2, sub2.arr2 a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}'
+1,'NULL'
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}'
+2,'{"inner_struct":null,"small":104}'
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Unnest an array that contains structs from a HMS view.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, item from collection_struct_mix_view,
+ collection_struct_mix_view.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}'
+1,'NULL'
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}'
+2,'{"inner_struct":null,"small":104}'
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}'
+2,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Doubly unnest two-level array from nested inline view, displaying the unnested results
+# at both levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix),
+ sub2 as (select id, struct_contains_nested_arr s from sub)
+select id, arr.item, inner_arr.item from sub2, sub2.s.arr arr, arr.item inner_arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-05
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-06
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',NULL
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-07
+1,'["2022-12-08","2022-12-09",null]',2022-12-08
+1,'["2022-12-08","2022-12-09",null]',2022-12-09
+1,'["2022-12-08","2022-12-09",null]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-10
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-11
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-12
+---- TYPES
+INT,STRING,DATE
+====
+---- QUERY
+# Doubly unnest two-level array from HMS view, displaying the unnested results at both
+# levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+select id, arr.item, inner_arr.item from collection_struct_mix_view,
+ collection_struct_mix_view.struct_contains_nested_arr.arr arr, arr.item inner_arr;
+---- RESULTS
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-05
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-06
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',NULL
+1,'["2022-12-05","2022-12-06",null,"2022-12-07"]',2022-12-07
+1,'["2022-12-08","2022-12-09",null]',2022-12-08
+1,'["2022-12-08","2022-12-09",null]',2022-12-09
+1,'["2022-12-08","2022-12-09",null]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-10
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-11
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',NULL
+2,'["2022-12-10","2022-12-11",null,"2022-12-12"]',2022-12-12
+---- TYPES
+INT,STRING,DATE
+====
+---- QUERY
+# Join unnest array containing struct and also query struct fields.
+select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix,
+ collection_struct_mix.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2
+1,'NULL','NULL',NULL
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20
+2,'{"inner_struct":null,"small":104}','NULL',104
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105
+2,'NULL','NULL',NULL
+---- TYPES
+INT,STRING,STRING,SMALLINT
+====
+---- QUERY
+# Join unnest array containing struct from HMS view and also query struct fields.
+select id, a.item, a.item.inner_struct, a.item.small from collection_struct_mix_view,
+ collection_struct_mix_view.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2
+1,'NULL','NULL',NULL
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20
+2,'{"inner_struct":null,"small":104}','NULL',104
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105
+2,'NULL','NULL',NULL
+---- TYPES
+INT,STRING,STRING,SMALLINT
+====
+---- QUERY
+# Join unnest array containing struct from inline view and also query struct fields.
+with sub as (select id, arr_contains_nested_struct from collection_struct_mix_view)
+select id, a.item, a.item.inner_struct, a.item.small from sub,
+ sub.arr_contains_nested_struct a;
+---- RESULTS
+1,'{"inner_struct":{"str":"","l":0},"small":2}','{"str":"","l":0}',2
+1,'NULL','NULL',NULL
+1,'{"inner_struct":{"str":"some_string","l":5},"small":20}','{"str":"some_string","l":5}',20
+2,'{"inner_struct":null,"small":104}','NULL',104
+2,'{"inner_struct":{"str":"aaa","l":28},"small":105}','{"str":"aaa","l":28}',105
+2,'NULL','NULL',NULL
+---- TYPES
+INT,STRING,STRING,SMALLINT
+====
+---- QUERY
+# Zipping unnest an array that contains a struct.
+select unnest(arr_contains_struct) from collection_struct_mix;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest on an array that contains a struct with FROM-clause unnest syntax.
+select a.item from collection_struct_mix,
+ unnest(collection_struct_mix.arr_contains_struct) as (a);
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest of two arrays that contain a structs.
+select unnest(arr_contains_struct), unnest(arr_contains_nested_struct)
+from collection_struct_mix;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest of two arrays that contain a structs, from view
+select unnest(arr_contains_struct), unnest(arr_contains_nested_struct)
+from collection_struct_mix_view;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest of two arrays that contain structs, from view.
+with unnesting as (
+ select unnest(arr_contains_struct) struct1, unnest(arr_contains_nested_struct) struct2
+ from collection_struct_mix_view)
+select struct1, struct2 from unnesting
+where struct1.i > 2;
+---- CATCH
+AnalysisException: Zipping unnest on an array that (recursively) contains a struct is not supported.
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported.
+select unnest(struct_contains_nested_arr.arr) from collection_struct_mix;
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported with FROM-clause unnest
+# syntax.
+select a.item from collection_struct_mix,
+ unnest(collection_struct_mix.struct_contains_nested_arr.arr) as (a);
+---- RESULTS
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+---- TYPES
+STRING
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported; querying from a HMS
+# view.
+select unnest(struct_contains_nested_arr.arr) from collection_struct_mix_view;
+---- RESULTS
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+---- TYPES
+STRING
+====
+---- QUERY
+# Zipping unnest on an array that is in a struct is not supported; querying from an inline
+# view.
+with sub as (select struct_contains_nested_arr from collection_struct_mix)
+select unnest(struct_contains_nested_arr.arr) from sub;
+---- RESULTS
+---- CATCH
+AnalysisException: Zipping unnest on an array that is within a struct is not supported.
+---- TYPES
+STRING
+====
+---- QUERY
+# Test that NULL map keys are printed correctly with STRINGIFY_MAP_KEYS=true.
+set STRINGIFY_MAP_KEYS=1;
+select id, struct_contains_map from collection_struct_mix;
+---- RESULTS
+1,'{"m":{"1":"one","2":"two","0":null}}'
+2,'{"m":null}'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix_view)
+select id, struct_contains_nested_arr.arr from (select id, struct_contains_nested_arr from sub) sub2
+where struct_contains_nested_arr.i > 4;
+---- RESULTS
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (select id, struct_contains_nested_arr from collection_struct_mix_view)
+select id, struct_contains_nested_arr.arr from
+ (select id, struct_contains_nested_arr
+ from sub
+ where struct_contains_nested_arr.i > 4) sub2;
+---- RESULTS
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, struct_contains_nested_arr
+ from collection_struct_mix_view
+ where struct_contains_nested_arr.i > 4)
+select id, struct_contains_nested_arr.arr from (select id, struct_contains_nested_arr from
+sub) sub2;
+---- RESULTS
+2,'[["2022-12-10","2022-12-11",null,"2022-12-12"],null]'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, struct_contains_nested_arr
+ from collection_struct_mix_view)
+select id, arr2.item from (
+ select id, arr.item single_arr
+ from sub, sub.struct_contains_nested_arr.arr arr) sub2, sub2.single_arr arr2
+where arr2.item > "2022-12-06";
+---- RESULTS
+1,2022-12-07
+1,2022-12-08
+1,2022-12-09
+2,2022-12-10
+2,2022-12-11
+2,2022-12-12
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, struct_contains_nested_arr
+ from collection_struct_mix_view)
+select id, arr2.item from (
+ select id, arr.item single_arr
+ from sub, sub.struct_contains_nested_arr.arr arr
+ where sub.struct_contains_nested_arr.i > 4) sub2, sub2.single_arr arr2
+where arr2.item > "2022-12-06";
+---- RESULTS
+2,2022-12-10
+2,2022-12-11
+2,2022-12-12
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Using different kinds of views and adding WHERE clauses at different levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with sub as (
+ select id, arr1.item single_arr
+ from collection_struct_mix, collection_struct_mix.struct_contains_nested_arr.arr arr1
+ where struct_contains_nested_arr.i > 4)
+select id, d from (
+ select id, arr2.item d
+ from sub, sub.single_arr arr2) sub2
+where d > "2022-12-06";
+---- RESULTS
+2,2022-12-10
+2,2022-12-11
+2,2022-12-12
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Using WHERE filters on the same column at different view levels.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+with project as (
+ select id, struct_contains_nested_arr.arr nested_array from collection_struct_mix_view
+),
+unnest_to_1d_array as (
+ select id, item array_1d from project, project.nested_array
+),
+unnest_to_scalars as (
+ select id, item scalar_item from unnest_to_1d_array, unnest_to_1d_array.array_1d
+ where item > "2022-12-05"
+)
+select id, scalar_item from unnest_to_scalars
+where scalar_item < "2022-12-11";
+---- RESULTS
+1,2022-12-06
+1,2022-12-07
+1,2022-12-08
+1,2022-12-09
+2,2022-12-10
+---- TYPES
+INT,DATE
+====
+---- QUERY
+# Test that complex types are propagated through views with EXPAND_COMPLEX_TYPES=1.
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=1;
+with sub as (
+ select * from collection_struct_mix)
+select * from (
+ select * from sub
+) sub2;
+---- RESULTS
+1,'{"arr":[1,2,3,4,null,null,5]}','{"m":{1:"one",2:"two",0:null}}','[{"i":1},{"i":2},{"i":3},{"i":4},null,{"i":5},{"i":null}]','[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','{"arr":[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]],"i":2}','{10:{"big":{"arr":[{"inner_arr":[[0,null,-1,-5,null,8],[20,null]],"m":"2022-12-05 14:30:00"},{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":" [...]
+2,'{"arr":null}','{"m":null}','[{"i":100},{"i":8},{"i":35},{"i":45},null,{"i":193},{"i":null}]','[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','{"arr":[["2022-12-10","2022-12-11",null,"2022-12-12"],null],"i":2754}','{20:{"big":{"arr":[null,{"inner_arr":[[12,1024,null],[null,null,84],[null,15,null]],"m":"2022-12-10 08:15:12"}],"n":95},"small":{"str":"otherstring","i":2048}},21:{"big":{"arr":null,"n":8},"small":{"str":"textstring","i":0}},22:{" [...]
+---- TYPES
+INT,STRING,STRING,STRING,STRING,STRING,STRING
+====
+---- QUERY
+# Test that struct elements can be star-exanded with EXPAND_COMPLEX_TYPES=0;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=0;
+with sub as (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from
+collection_struct_mix)
+select id, arr_contains_nested_struct, struct_contains_nested_arr.* from (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from sub
+) sub2;
+---- RESULTS
+1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]',2
+2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]',2754
+---- TYPES
+INT,STRING,INT
+====
+---- QUERY
+# Test that struct elements can be star-exanded with EXPAND_COMPLEX_TYPES=1;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=1;
+with sub as (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from
+collection_struct_mix)
+select id, arr_contains_nested_struct, struct_contains_nested_arr.* from (
+ select id, arr_contains_nested_struct, struct_contains_nested_arr from sub
+) sub2;
+---- RESULTS
+1,'[{"inner_struct":{"str":"","l":0},"small":2},null,{"inner_struct":{"str":"some_string","l":5},"small":20}]','[["2022-12-05","2022-12-06",null,"2022-12-07"],["2022-12-08","2022-12-09",null]]',2
+2,'[{"inner_struct":null,"small":104},{"inner_struct":{"str":"aaa","l":28},"small":105},null]','[["2022-12-10","2022-12-11",null,"2022-12-12"],null]',2754
+---- TYPES
+INT,STRING,STRING,INT
+====
+---- QUERY
+# Test that struct elements from unnested array can be star-exanded with
+# EXPAND_COMPLEX_TYPES=0;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=0;
+select id, arr.item.*
+from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr;
+---- RESULTS
+1,2
+1,NULL
+1,20
+2,104
+2,105
+2,NULL
+---- TYPES
+INT,SMALLINT
+====
+---- QUERY
+# Test that struct elements from unnested array can be star-exanded with
+# EXPAND_COMPLEX_TYPES=1;
+set CONVERT_LEGACY_HIVE_PARQUET_UTC_TIMESTAMPS=1;
+set EXPAND_COMPLEX_TYPES=1;
+select id, arr.item.*
+from collection_struct_mix, collection_struct_mix.arr_contains_nested_struct arr;
+---- RESULTS
+1,'{"str":"","l":0}',2
+1,'NULL',NULL
+1,'{"str":"some_string","l":5}',20
+2,'NULL',104
+2,'{"str":"aaa","l":28}',105
+2,'NULL',NULL
+---- TYPES
+INT,STRING,SMALLINT
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
index 6e7b57935..24bf1b20e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
@@ -79,7 +79,7 @@ INT
set EXPAND_COMPLEX_TYPES=1;
select nested_struct.* from complextypestbl
---- CATCH
-AnalysisException: Struct containing a collection type is not allowed in the select list.
+AnalysisException: Struct type in select list is not allowed when Codegen is ON. You might want to set DISABLE_CODEGEN=true
====
---- QUERY
# Test resolving explicit STAR path on a nested struct column inside array
diff --git a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
index f2b5ed498..082892649 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
@@ -597,16 +597,6 @@ where alltypes in (select alltypes from functional_parquet.complextypes_structs)
AnalysisException: A subquery can't return complex types. (SELECT alltypes FROM functional_parquet.complextypes_structs)
====
---- QUERY
-select tbl.nested_struct from complextypestbl tbl;
----- CATCH
-AnalysisException: Struct containing a collection type is not allowed in the select list.
-====
----- QUERY
-select tbl.nested_struct.c from complextypestbl tbl;
----- CATCH
-AnalysisException: Struct containing a collection type is not allowed in the select list.
-====
----- QUERY
# Unioning structs is not supported.
# IMPALA-10752
select id, tiny_struct from complextypes_structs
@@ -636,15 +626,3 @@ order by 3
---- CATCH
AnalysisException: ORDER BY: ordinal exceeds the number of items in the SELECT list: 3
====
----- QUERY
-# Structs inside arrays are not yet supported.
-select nested_struct.c.d from complextypestbl;
----- CATCH
-AnalysisException: STRUCT type inside collection types is not supported.
-====
----- QUERY
-# Structs inside maps are not yet supported.
-select nested_struct.g from complextypestbl;
----- CATCH
-AnalysisException: STRUCT type inside collection types is not supported.
-====
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 999989440..99d44e537 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -182,9 +182,41 @@ class TestNestedCollectionsInSelectList(ImpalaTestSuite):
"""Queries where a map has null keys. Is only possible in ORC, not Parquet."""
if vector.get_value('table_format').file_format == 'parquet':
pytest.skip()
+ # Structs in select list are not supported with codegen enabled: see IMPALA-10851.
+ if vector.get_value('exec_option')['disable_codegen'] == 'False':
+ pytest.skip()
self.run_test_case('QueryTest/map_null_keys', vector)
+class TestMixedCollectionsAndStructsInSelectList(ImpalaTestSuite):
+ """Functional tests for the case where collections and structs are embedded into one
+ another and they are provided in the select list."""
+ @classmethod
+ def get_workload(self):
+ return 'functional-query'
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestMixedCollectionsAndStructsInSelectList, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ v.get_value('table_format').file_format in ['parquet', 'orc'])
+ cls.ImpalaTestMatrix.add_dimension(
+ ImpalaTestDimension('mt_dop', 0, 2))
+ cls.ImpalaTestMatrix.add_dimension(
+ create_exec_option_dimension_from_dict({
+ 'disable_codegen': ['False', 'True']}))
+ cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
+ cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('orc_schema_resolution', 0, 1))
+ cls.ImpalaTestMatrix.add_constraint(orc_schema_resolution_constraint)
+
+ def test_mixed_complex_types_in_select_list(self, vector, unique_database):
+ """Queries where structs and collections are embedded into one another."""
+ # Structs in select list are not supported with codegen enabled: see IMPALA-10851.
+ if vector.get_value('exec_option')['disable_codegen'] == 'False':
+ pytest.skip()
+ self.run_test_case('QueryTest/mixed-collections-and-structs', vector)
+
+
class TestComputeStatsWithNestedTypes(ImpalaTestSuite):
"""Functional tests for running compute stats on tables that have nested types in the
columns."""
[impala] 01/03: IMPALA-11479: Add Java unit tests for IcebergUtil.
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 374d011a7caae3ad91e209f39b4537f9152a8ca2
Author: Andrew Sherman <as...@cloudera.com>
AuthorDate: Fri Aug 5 14:20:21 2022 -0700
IMPALA-11479: Add Java unit tests for IcebergUtil.
This does not test all of IcebergUtil, but it is a start.
Tidy up some IcebergUtil code and fix a few spelling mistakes.
Change-Id: Ib15993a9ed3d5802dda4edb2011a90ead6d06ed4
Reviewed-on: http://gerrit.cloudera.org:8080/19543
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/common/TransactionKeepalive.java | 1 -
.../java/org/apache/impala/util/IcebergUtil.java | 113 +++---
.../impala/catalog/local/LocalCatalogTest.java | 2 +-
.../org/apache/impala/util/IcebergUtilTest.java | 400 +++++++++++++++++++++
4 files changed, 460 insertions(+), 56 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/common/TransactionKeepalive.java b/fe/src/main/java/org/apache/impala/common/TransactionKeepalive.java
index 52dff28fd..126937826 100644
--- a/fe/src/main/java/org/apache/impala/common/TransactionKeepalive.java
+++ b/fe/src/main/java/org/apache/impala/common/TransactionKeepalive.java
@@ -37,7 +37,6 @@ import org.apache.impala.thrift.TQueryCtx;
import org.apache.log4j.Logger;
import com.google.common.base.Preconditions;
-import com.sun.tools.javac.code.Attribute.Array;
/**
* Object of this class creates a daemon thread that periodically heartbeats the
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index 5dbbf61f2..c6595a968 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -99,10 +99,13 @@ import org.apache.impala.thrift.TIcebergPartitionField;
import org.apache.impala.thrift.TIcebergPartitionSpec;
import org.apache.impala.thrift.TIcebergPartitionTransformType;
+@SuppressWarnings("UnstableApiUsage")
public class IcebergUtil {
private static final int ICEBERG_EPOCH_YEAR = 1970;
private static final int ICEBERG_EPOCH_MONTH = 1;
+ @SuppressWarnings("unused")
private static final int ICEBERG_EPOCH_DAY = 1;
+ @SuppressWarnings("unused")
private static final int ICEBERG_EPOCH_HOUR = 0;
/**
@@ -124,8 +127,7 @@ public class IcebergUtil {
case HIVE_CATALOG: return IcebergHiveCatalog.getInstance();
case HADOOP_CATALOG: return new IcebergHadoopCatalog(location);
case CATALOGS: return IcebergCatalogs.getInstance();
- default: throw new ImpalaRuntimeException (
- "Unexpected catalog type: " + catalog.toString());
+ default: throw new ImpalaRuntimeException("Unexpected catalog type: " + catalog);
}
}
@@ -186,8 +188,7 @@ public class IcebergUtil {
* Get Iceberg Transaction for 'feTable', usually use Transaction to update Iceberg
* table schema.
*/
- public static Transaction getIcebergTransaction(FeIcebergTable feTable)
- throws TableLoadingException, ImpalaRuntimeException {
+ public static Transaction getIcebergTransaction(FeIcebergTable feTable) {
return feTable.getIcebergApiTable().newTransaction();
}
@@ -317,7 +318,7 @@ public class IcebergUtil {
*/
public static TIcebergFileFormat getIcebergFileFormat(
org.apache.hadoop.hive.metastore.api.Table msTable) {
- TIcebergFileFormat fileFormat = null;
+ TIcebergFileFormat fileFormat;
Map<String, String> params = msTable.getParameters();
if (params.containsKey(IcebergTable.ICEBERG_FILE_FORMAT)) {
fileFormat = IcebergUtil.getIcebergFileFormat(
@@ -389,7 +390,7 @@ public class IcebergUtil {
PartitionField field, HashMap<String, Integer> transformParams)
throws TableLoadingException {
String type = field.transform().toString();
- String transformMappingKey = getPartitonTransformMappingKey(field.sourceId(),
+ String transformMappingKey = getPartitionTransformMappingKey(field.sourceId(),
getPartitionTransformType(type));
return getPartitionTransform(type, transformParams.get(transformMappingKey));
}
@@ -405,15 +406,15 @@ public class IcebergUtil {
return getPartitionTransform(transformType, null);
}
- public static TIcebergPartitionTransformType getPartitionTransformType(
+ private static TIcebergPartitionTransformType getPartitionTransformType(
String transformType) throws TableLoadingException {
Preconditions.checkNotNull(transformType);
transformType = transformType.toUpperCase();
if ("IDENTITY".equals(transformType)) {
return TIcebergPartitionTransformType.IDENTITY;
- } else if (transformType != null && transformType.startsWith("BUCKET")) {
+ } else if (transformType.startsWith("BUCKET")) {
return TIcebergPartitionTransformType.BUCKET;
- } else if (transformType != null && transformType.startsWith("TRUNCATE")) {
+ } else if (transformType.startsWith("TRUNCATE")) {
return TIcebergPartitionTransformType.TRUNCATE;
}
switch (transformType) {
@@ -428,8 +429,8 @@ public class IcebergUtil {
}
}
- private static String getPartitonTransformMappingKey(int sourceId,
- TIcebergPartitionTransformType transformType) {
+ private static String getPartitionTransformMappingKey(
+ int sourceId, TIcebergPartitionTransformType transformType) {
return sourceId + "_" + transformType.toString();
}
@@ -438,10 +439,10 @@ public class IcebergUtil {
* PartitionSpec and its transform's parameter. Only Bucket and Truncate transforms
* have a parameter, for other transforms this mapping will have a null.
* source ID and the transform type are needed together to uniquely identify a specific
- * field in the PartitionSpec. (Unfortunaltely, fieldId is not available in the Visitor
+ * field in the PartitionSpec. (Unfortunately, fieldId is not available in the Visitor
* class below.)
* The reason for implementing the PartitionSpecVisitor below was that Iceberg doesn't
- * expose the interface of the transform types outside of their package and the only
+ * expose the interface of the transform types outside their package and the only
* way to get the transform's parameter is implementing this visitor class.
*/
public static HashMap<String, Integer> getPartitionTransformParams(PartitionSpec spec) {
@@ -449,61 +450,61 @@ public class IcebergUtil {
spec, new PartitionSpecVisitor<Pair<String, Integer>>() {
@Override
public Pair<String, Integer> identity(String sourceName, int sourceId) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.IDENTITY);
- return new Pair<String, Integer>(mappingKey, null);
+ return new Pair<>(mappingKey, null);
}
@Override
public Pair<String, Integer> bucket(String sourceName, int sourceId,
int numBuckets) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.BUCKET);
- return new Pair<String, Integer>(mappingKey, numBuckets);
+ return new Pair<>(mappingKey, numBuckets);
}
@Override
public Pair<String, Integer> truncate(String sourceName, int sourceId,
int width) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.TRUNCATE);
- return new Pair<String, Integer>(mappingKey, width);
+ return new Pair<>(mappingKey, width);
}
@Override
public Pair<String, Integer> year(String sourceName, int sourceId) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.YEAR);
- return new Pair<String, Integer>(mappingKey, null);
+ return new Pair<>(mappingKey, null);
}
@Override
public Pair<String, Integer> month(String sourceName, int sourceId) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.MONTH);
- return new Pair<String, Integer>(mappingKey, null);
+ return new Pair<>(mappingKey, null);
}
@Override
public Pair<String, Integer> day(String sourceName, int sourceId) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.DAY);
- return new Pair<String, Integer>(mappingKey, null);
+ return new Pair<>(mappingKey, null);
}
@Override
public Pair<String, Integer> hour(String sourceName, int sourceId) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.HOUR);
- return new Pair<String, Integer>(mappingKey, null);
+ return new Pair<>(mappingKey, null);
}
@Override
public Pair<String, Integer> alwaysNull(int fieldId, String sourceName,
int sourceId) {
- String mappingKey = getPartitonTransformMappingKey(sourceId,
+ String mappingKey = getPartitionTransformMappingKey(sourceId,
TIcebergPartitionTransformType.VOID);
- return new Pair<String, Integer>(mappingKey, null);
+ return new Pair<>(mappingKey, null);
}
});
// Move the content of the List into a HashMap for faster querying in the future.
@@ -539,8 +540,14 @@ public class IcebergUtil {
/**
* Transform TIcebergFileFormat to HdfsFileFormat
*/
+ @SuppressWarnings("unused")
public static HdfsFileFormat toHdfsFileFormat(String format) {
- return HdfsFileFormat.fromThrift(toTHdfsFileFormat(getIcebergFileFormat(format)));
+ TIcebergFileFormat icebergFileFormat = getIcebergFileFormat(format);
+ if (icebergFileFormat == null) {
+ // Can't pass null to toTHdfsFileFormat(), so throw.
+ throw new IllegalArgumentException("unknown table format " + format);
+ }
+ return HdfsFileFormat.fromThrift(toTHdfsFileFormat(icebergFileFormat));
}
/**
@@ -645,7 +652,6 @@ public class IcebergUtil {
}
@Override
- @SuppressWarnings("unchecked")
public <T> T get(int pos, Class<T> javaClass) {
return javaClass.cast(values[pos]);
}
@@ -741,7 +747,7 @@ public class IcebergUtil {
* return value should be 14.
*/
private static Integer parseYearToTransformYear(String yearStr) {
- Integer year = Integer.valueOf(yearStr);
+ int year = Integer.parseInt(yearStr);
return year - ICEBERG_EPOCH_YEAR;
}
@@ -749,12 +755,11 @@ public class IcebergUtil {
* In the partition path months are represented as <year>-<month>, e.g. 2021-01. We
* need to convert it to a single integer which represents the months from '1970-01'.
*/
- private static Integer parseMonthToTransformMonth(String monthStr)
- throws ImpalaRuntimeException {
+ private static Integer parseMonthToTransformMonth(String monthStr) {
String[] parts = monthStr.split("-", -1);
Preconditions.checkState(parts.length == 2);
- Integer year = Integer.valueOf(parts[0]);
- Integer month = Integer.valueOf(parts[1]);
+ int year = Integer.parseInt(parts[0]);
+ int month = Integer.parseInt(parts[1]);
int years = year - ICEBERG_EPOCH_YEAR;
int months = month - ICEBERG_EPOCH_MONTH;
return years * 12 + months;
@@ -769,10 +774,10 @@ public class IcebergUtil {
final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC);
String[] parts = hourStr.split("-", -1);
Preconditions.checkState(parts.length == 4);
- Integer year = Integer.valueOf(parts[0]);
- Integer month = Integer.valueOf(parts[1]);
- Integer day = Integer.valueOf(parts[2]);
- Integer hour = Integer.valueOf(parts[3]);
+ int year = Integer.parseInt(parts[0]);
+ int month = Integer.parseInt(parts[1]);
+ int day = Integer.parseInt(parts[2]);
+ int hour = Integer.parseInt(parts[3]);
OffsetDateTime datetime = OffsetDateTime.of(
LocalDateTime.of(year, month, day, hour, /*minute=*/0),
ZoneOffset.UTC);
@@ -812,9 +817,9 @@ public class IcebergUtil {
clevel > IcebergTable.MAX_PARQUET_COMPRESSION_LEVEL) {
errMsg.append("Parquet compression level for Iceberg table should fall in " +
"the range of [")
- .append(String.valueOf(IcebergTable.MIN_PARQUET_COMPRESSION_LEVEL))
+ .append(IcebergTable.MIN_PARQUET_COMPRESSION_LEVEL)
.append("..")
- .append(String.valueOf(IcebergTable.MAX_PARQUET_COMPRESSION_LEVEL))
+ .append(IcebergTable.MAX_PARQUET_COMPRESSION_LEVEL)
.append("]");
return null;
}
@@ -836,9 +841,9 @@ public class IcebergUtil {
rowGroupSize > IcebergTable.MAX_PARQUET_ROW_GROUP_SIZE) {
errMsg.append("Parquet row group size for Iceberg table should ")
.append("fall in the range of [")
- .append(String.valueOf(IcebergTable.MIN_PARQUET_ROW_GROUP_SIZE))
+ .append(IcebergTable.MIN_PARQUET_ROW_GROUP_SIZE)
.append("..")
- .append(String.valueOf(IcebergTable.MAX_PARQUET_ROW_GROUP_SIZE))
+ .append(IcebergTable.MAX_PARQUET_ROW_GROUP_SIZE)
.append("]");
return null;
}
@@ -873,9 +878,9 @@ public class IcebergUtil {
errMsg.append("Parquet ")
.append(descr)
.append(" for Iceberg table should fall in the range of [")
- .append(String.valueOf(IcebergTable.MIN_PARQUET_PAGE_SIZE))
+ .append(IcebergTable.MIN_PARQUET_PAGE_SIZE)
.append("..")
- .append(String.valueOf(IcebergTable.MAX_PARQUET_PAGE_SIZE))
+ .append(IcebergTable.MAX_PARQUET_PAGE_SIZE)
.append("]");
return null;
}
@@ -979,45 +984,45 @@ public class IcebergUtil {
schema, field, new PartitionSpecVisitor<Pair<Byte, Integer>>() {
@Override
public Pair<Byte, Integer> identity(String sourceName, int sourceId) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.IDENTITY, null);
+ return new Pair<>(FbIcebergTransformType.IDENTITY, null);
}
@Override
public Pair<Byte, Integer> bucket(String sourceName, int sourceId,
int numBuckets) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.BUCKET, numBuckets);
+ return new Pair<>(FbIcebergTransformType.BUCKET, numBuckets);
}
@Override
public Pair<Byte, Integer> truncate(String sourceName, int sourceId,
int width) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.TRUNCATE, width);
+ return new Pair<>(FbIcebergTransformType.TRUNCATE, width);
}
@Override
public Pair<Byte, Integer> year(String sourceName, int sourceId) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.YEAR, null);
+ return new Pair<>(FbIcebergTransformType.YEAR, null);
}
@Override
public Pair<Byte, Integer> month(String sourceName, int sourceId) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.MONTH, null);
+ return new Pair<>(FbIcebergTransformType.MONTH, null);
}
@Override
public Pair<Byte, Integer> day(String sourceName, int sourceId) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.DAY, null);
+ return new Pair<>(FbIcebergTransformType.DAY, null);
}
@Override
public Pair<Byte, Integer> hour(String sourceName, int sourceId) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.HOUR, null);
+ return new Pair<>(FbIcebergTransformType.HOUR, null);
}
@Override
public Pair<Byte, Integer> alwaysNull(int fieldId, String sourceName,
int sourceId) {
- return new Pair<Byte, Integer>(FbIcebergTransformType.VOID, null);
+ return new Pair<>(FbIcebergTransformType.VOID, null);
}
});
}
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index 5f6e78fd2..a2a72ce3c 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -272,7 +272,7 @@ public class LocalCatalogTest {
}
/**
- * This test verifies that the network adresses used by the LocalIcebergTable are
+ * This test verifies that the network addresses used by the LocalIcebergTable are
* the same used by CatalogD.
*/
@Test
diff --git a/fe/src/test/java/org/apache/impala/util/IcebergUtilTest.java b/fe/src/test/java/org/apache/impala/util/IcebergUtilTest.java
new file mode 100644
index 000000000..3a3d0031c
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/util/IcebergUtilTest.java
@@ -0,0 +1,400 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+package org.apache.impala.util;
+
+import static org.apache.impala.thrift.TIcebergCatalog.CATALOGS;
+import static org.apache.impala.thrift.TIcebergCatalog.HADOOP_CATALOG;
+import static org.apache.impala.thrift.TIcebergCatalog.HADOOP_TABLES;
+import static org.apache.impala.thrift.TIcebergCatalog.HIVE_CATALOG;
+import static org.apache.impala.util.IcebergUtil.getFilePathHash;
+import static org.apache.impala.util.IcebergUtil.getIcebergFileFormat;
+import static org.apache.impala.util.IcebergUtil.getPartitionTransform;
+import static org.apache.impala.util.IcebergUtil.getPartitionTransformParams;
+import static org.apache.impala.util.IcebergUtil.isPartitionColumn;
+import static org.apache.impala.util.IcebergUtil.toHdfsFileFormat;
+import static org.apache.impala.util.IcebergUtil.toTHdfsFileFormat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.mr.Catalogs;
+import org.apache.iceberg.types.Types;
+import org.apache.impala.analysis.IcebergPartitionField;
+import org.apache.impala.analysis.IcebergPartitionSpec;
+import org.apache.impala.analysis.IcebergPartitionTransform;
+import org.apache.impala.catalog.HdfsFileFormat;
+import org.apache.impala.catalog.IcebergColumn;
+import org.apache.impala.catalog.IcebergTable;
+import org.apache.impala.catalog.TableLoadingException;
+import org.apache.impala.catalog.Type;
+import org.apache.impala.catalog.iceberg.IcebergCatalog;
+import org.apache.impala.catalog.iceberg.IcebergCatalogs;
+import org.apache.impala.catalog.iceberg.IcebergHadoopCatalog;
+import org.apache.impala.catalog.iceberg.IcebergHadoopTables;
+import org.apache.impala.catalog.iceberg.IcebergHiveCatalog;
+import org.apache.impala.common.AnalysisException;
+import org.apache.impala.common.ImpalaRuntimeException;
+import org.apache.impala.thrift.THdfsFileFormat;
+import org.apache.impala.thrift.TIcebergCatalog;
+import org.apache.impala.thrift.TIcebergFileFormat;
+import org.apache.impala.thrift.TIcebergPartitionTransformType;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Unit tests for Iceberg Utilities.
+ */
+public class IcebergUtilTest {
+ /**
+ * Unit test for IcebergUtil.getTIcebergCatalog() and IcebergUtil.getIcebergCatalog().
+ */
+ @Test
+ public void testGetCatalog() throws ImpalaRuntimeException {
+ CatalogMapping[] mappings = new CatalogMapping[] {
+ new CatalogMapping("hadoop.tables", HADOOP_TABLES, IcebergHadoopTables.class),
+ new CatalogMapping("hadoop.catalog", HADOOP_CATALOG, IcebergHadoopCatalog.class),
+ new CatalogMapping("hive.catalog", HIVE_CATALOG, IcebergHiveCatalog.class),
+ new CatalogMapping(null, HIVE_CATALOG, IcebergHiveCatalog.class),
+ new CatalogMapping("other string", CATALOGS, IcebergCatalogs.class),
+ };
+ for (CatalogMapping testValue : mappings) {
+ TIcebergCatalog catalog = IcebergUtil.getTIcebergCatalog(testValue.propertyName);
+ assertEquals("err for " + testValue.propertyName, testValue.catalog, catalog);
+ IcebergCatalog impl = IcebergUtil.getIcebergCatalog(catalog, "location");
+ assertEquals("err for " + testValue.propertyName, testValue.clazz, impl.getClass());
+ }
+ }
+
+ /**
+ * Unit test for IcebergUtil.getIcebergTableIdentifier().
+ */
+ @Test
+ public void testGetIcebergTableIdentifier() {
+ // Test a table with no table properties.
+ Table table = new Table();
+ table.setParameters(new HashMap<>());
+ String tableName = "table_name";
+ table.setTableName(tableName);
+ String dbname = "database_name";
+ table.setDbName(dbname);
+ TableIdentifier icebergTableIdentifier = IcebergUtil.getIcebergTableIdentifier(table);
+ assertEquals(
+ TableIdentifier.parse("database_name.table_name"), icebergTableIdentifier);
+
+ // If iceberg.table_identifier is not set then the value of the "name" property
+ // is used.
+ String nameId = "db.table";
+ table.putToParameters(Catalogs.NAME, nameId);
+ icebergTableIdentifier = IcebergUtil.getIcebergTableIdentifier(table);
+ assertEquals(TableIdentifier.parse(nameId), icebergTableIdentifier);
+
+ // If iceberg.table_identifier is set then that is used.
+ String tableId = "foo.bar";
+ table.putToParameters(IcebergTable.ICEBERG_TABLE_IDENTIFIER, tableId);
+ icebergTableIdentifier = IcebergUtil.getIcebergTableIdentifier(table);
+ assertEquals(TableIdentifier.parse(tableId), icebergTableIdentifier);
+
+ // If iceberg.table_identifier set to a simple name, then the default catalog is used.
+ table.putToParameters(IcebergTable.ICEBERG_TABLE_IDENTIFIER, "noDatabase");
+ icebergTableIdentifier = IcebergUtil.getIcebergTableIdentifier(table);
+ assertEquals(TableIdentifier.parse("default.noDatabase"), icebergTableIdentifier);
+ }
+
+ /**
+ * Unit test for isHiveCatalog().
+ */
+ @Test
+ public void testIsHiveCatalog() {
+ CatalogType[] catalogTypes = new CatalogType[] {
+ // For hadoop.tables amd hadoop.catalog we are not using Hive Catalog.
+ new CatalogType("hadoop.tables", false),
+ new CatalogType("hadoop.catalog", false),
+ // For all other values of ICEBERG_CATALOG then Hive Catalog is used.
+ new CatalogType("hive.catalog", true),
+ new CatalogType(null, true),
+ new CatalogType("other string", true),
+ };
+ for (CatalogType testValue : catalogTypes) {
+ Table table = new Table();
+ table.putToParameters(IcebergTable.ICEBERG_CATALOG, testValue.propertyName);
+ assertEquals("err in " + testValue.propertyName, testValue.isHiveCatalog,
+ IcebergUtil.isHiveCatalog(table));
+ }
+ }
+
+ /**
+ * Unit test for getIcebergFileFormat(), toHdfsFileFormat() and toTHdfsFileFormat().
+ */
+ @Test
+ public void testToHdfsFileFormat() {
+ assertEquals(THdfsFileFormat.ORC, toTHdfsFileFormat(TIcebergFileFormat.ORC));
+ assertEquals(THdfsFileFormat.PARQUET, toTHdfsFileFormat(TIcebergFileFormat.PARQUET));
+ assertEquals(HdfsFileFormat.ORC, toHdfsFileFormat(TIcebergFileFormat.ORC));
+ assertEquals(HdfsFileFormat.PARQUET, toHdfsFileFormat(TIcebergFileFormat.PARQUET));
+ assertEquals(HdfsFileFormat.ORC, toHdfsFileFormat("ORC"));
+ assertEquals(HdfsFileFormat.PARQUET, toHdfsFileFormat("PARQUET"));
+ assertEquals(HdfsFileFormat.PARQUET, toHdfsFileFormat((String) null));
+ try {
+ toHdfsFileFormat("unknown");
+ fail("did not get expected assertion");
+ } catch (IllegalArgumentException e) {
+ // fall through
+ }
+ assertEquals(TIcebergFileFormat.ORC, getIcebergFileFormat("ORC"));
+ assertEquals(TIcebergFileFormat.PARQUET, getIcebergFileFormat("PARQUET"));
+ assertNull(getIcebergFileFormat("unknown"));
+ }
+
+ /**
+ * Unit test forgetPartitionTransform().
+ */
+ @Test
+ public void testGetPartitionTransform() {
+ // Case 1
+ // Transforms that work OK.
+ PartitionTransform[] goodTransforms = new PartitionTransform[] {
+ new PartitionTransform("BUCKET", 5),
+ new PartitionTransform("TRUNCATE", 4),
+ new PartitionTransform("HOUR", null),
+ new PartitionTransform("HOURS", null),
+ new PartitionTransform("DAY", null),
+ new PartitionTransform("DAYS", null),
+ new PartitionTransform("MONTH", null),
+ new PartitionTransform("MONTHS", null),
+ new PartitionTransform("YEAR", null),
+ new PartitionTransform("YEARS", null),
+ new PartitionTransform("VOID", null),
+ new PartitionTransform("IDENTITY", null),
+ };
+ for (PartitionTransform partitionTransform : goodTransforms) {
+ IcebergPartitionTransform transform = null;
+ try {
+ transform = getPartitionTransform(
+ partitionTransform.transformName, partitionTransform.parameter);
+ } catch (TableLoadingException t) {
+ fail("Transform " + partitionTransform + " caught unexpected " + t);
+ }
+ assertNotNull(transform);
+ try {
+ transform.analyze(null);
+ } catch (AnalysisException t) {
+ fail("Transform " + partitionTransform + " caught unexpected " + t);
+ }
+ }
+
+ // Case 2
+ // Transforms that get TableLoadingException.
+ PartitionTransform[] tableExceptions = new PartitionTransform[] {
+ new PartitionTransform("JUNK", -5),
+ };
+ for (PartitionTransform partitionTransform : tableExceptions) {
+ try {
+ /* IcebergPartitionTransform transform = */ getPartitionTransform(
+ partitionTransform.transformName, partitionTransform.parameter);
+ fail("Transform " + partitionTransform + " should have got exception");
+ } catch (TableLoadingException t) {
+ // OK, fall through
+ }
+ }
+
+ // Case 3
+ // Transforms that fail analysis.
+ PartitionTransform[] failAnalysis = new PartitionTransform[] {
+ new PartitionTransform("BUCKET", -5),
+ new PartitionTransform("TRUNCATE", -4),
+ };
+ for (PartitionTransform partitionTransform : failAnalysis) {
+ IcebergPartitionTransform transform = null;
+ try {
+ transform = getPartitionTransform(
+ partitionTransform.transformName, partitionTransform.parameter);
+ } catch (TableLoadingException t) {
+ fail("Transform " + partitionTransform + " caught unexpected " + t);
+ }
+ assertNotNull(transform);
+ try {
+ transform.analyze(null);
+ fail("Transform " + partitionTransform + " should have got exception");
+ } catch (AnalysisException t) {
+ // OK, fall through
+ }
+ }
+ }
+
+ /**
+ * Unit test for getDataFilePathHash().
+ */
+ @Test
+ public void testGetDataFilePathHash() {
+ String hash = getFilePathHash(FILE_A);
+ assertNotNull(hash);
+ String hash2 = getFilePathHash(FILE_A);
+ assertEquals(hash, hash2);
+ }
+
+ /**
+ * Unit test for getPartitionTransformParams().
+ */
+ @Test
+ public void testGetPartitionTransformParams() {
+ int numBuckets = 128;
+ PartitionSpec partitionSpec =
+ PartitionSpec.builderFor(SCHEMA).bucket("i", numBuckets).build();
+ HashMap<String, Integer> partitionTransformParams =
+ getPartitionTransformParams(partitionSpec);
+ assertNotNull(partitionTransformParams);
+ String expectedKey = "1_BUCKET";
+ assertTrue(partitionTransformParams.containsKey(expectedKey));
+ assertEquals(numBuckets, (long) partitionTransformParams.get(expectedKey));
+ }
+
+ /**
+ * Unit test for isPartitionColumn().
+ */
+ @Test
+ public void testIsPartitionColumn() {
+ {
+ // Case 1
+ // No partition fields: isPartitionColumn() should return false.
+ int fieldId = 3;
+ IcebergColumn column =
+ new IcebergColumn("name", Type.BOOLEAN, "comment", 0, fieldId, 5, 0, true);
+ List<IcebergPartitionField> fieldList = new ArrayList<>();
+ IcebergPartitionSpec icebergPartitionSpec = new IcebergPartitionSpec(4, fieldList);
+ assertFalse(isPartitionColumn(column, icebergPartitionSpec));
+ }
+ {
+ // Case 2
+ // A partition field source id matches a column field id: isPartitionColumn() should
+ // return true.
+ int id = 3;
+ IcebergColumn column =
+ new IcebergColumn("name", Type.BOOLEAN, "comment", 0, id, 105, 0, true);
+ IcebergPartitionTransform icebergPartitionTransform =
+ new IcebergPartitionTransform(TIcebergPartitionTransformType.IDENTITY);
+ IcebergPartitionField field =
+ new IcebergPartitionField(id, 106, "name", "name", icebergPartitionTransform);
+ ImmutableList<IcebergPartitionField> fieldList = ImmutableList.of(field);
+ IcebergPartitionSpec icebergPartitionSpec = new IcebergPartitionSpec(4, fieldList);
+ assertTrue(isPartitionColumn(column, icebergPartitionSpec));
+ }
+ {
+ // Case 3
+ // Partition field source id does not match a column field id: isPartitionColumn()
+ // should return false.
+ IcebergColumn column =
+ new IcebergColumn("name", Type.BOOLEAN, "comment", 0, 108, 105, 0, true);
+ IcebergPartitionTransform icebergPartitionTransform =
+ new IcebergPartitionTransform(TIcebergPartitionTransformType.IDENTITY);
+ IcebergPartitionField field =
+ new IcebergPartitionField(107, 106, "name", "name", icebergPartitionTransform);
+ ImmutableList<IcebergPartitionField> fieldList = ImmutableList.of(field);
+ IcebergPartitionSpec icebergPartitionSpec = new IcebergPartitionSpec(4, fieldList);
+ assertFalse(isPartitionColumn(column, icebergPartitionSpec));
+ }
+ }
+
+ /**
+ * Holder class for testing Partition transforms.
+ */
+ static class PartitionTransform {
+ String transformName;
+ Integer parameter;
+
+ PartitionTransform(String transformName, Integer parameter) {
+ this.transformName = transformName;
+ this.parameter = parameter;
+ }
+
+ @Override
+ public String toString() {
+ return "PartitionTransform{"
+ + "transformName='" + transformName + '\'' + ", parameter=" + parameter + '}';
+ }
+ }
+
+ /**
+ * A simple Schema object.
+ */
+ public static final Schema SCHEMA =
+ new Schema(Types.NestedField.required(1, "i", Types.IntegerType.get()),
+ Types.NestedField.required(2, "l", Types.LongType.get()),
+ Types.NestedField.required(3, "id", Types.IntegerType.get()),
+ Types.NestedField.required(4, "data", Types.StringType.get()));
+
+ /**
+ * Partition spec used to create tables.
+ */
+ protected static final PartitionSpec SPEC =
+ PartitionSpec.builderFor(SCHEMA).bucket("data", 10).build();
+
+ /**
+ * A test DataFile.
+ */
+ static final DataFile FILE_A =
+ DataFiles.builder(SPEC)
+ .withPath("/path/to/data-a.parquet")
+ .withFileSizeInBytes(10)
+ .withPartitionPath("data_bucket=0") // Easy way to set partition data for now.
+ .withRecordCount(1)
+ .build();
+
+ /**
+ * Holder class for testing isHiveCatalog().
+ */
+ static class CatalogType {
+ String propertyName;
+ boolean isHiveCatalog;
+
+ CatalogType(String propertyName, boolean isHiveCatalog) {
+ this.propertyName = propertyName;
+ this.isHiveCatalog = isHiveCatalog;
+ }
+ }
+
+ /**
+ * Holder class for test of catalog functions.
+ */
+ static class CatalogMapping {
+ String propertyName;
+ TIcebergCatalog catalog;
+ Class<?> clazz;
+
+ CatalogMapping(String propertyName, TIcebergCatalog catalog, Class<?> clazz) {
+ this.propertyName = propertyName;
+ this.catalog = catalog;
+ this.clazz = clazz;
+ }
+ }
+}
\ No newline at end of file