You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by la...@apache.org on 2026/07/22 19:12:00 UTC
(impala) branch master updated (01ca03c31 -> c4fe178ba)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 01ca03c31 IMPALA-14982: Add Impala-on-K8s deployment guide
new 407be0e24 IMPALA-14156: [DOCS] Documentation for query options
new 848ef4735 IMPALA-2210: Make Parquet the default file format
new c4fe178ba IMPALA-15054: Set EXPAND_COMPLEX_TYPES to true by default
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
common/thrift/Query.thrift | 4 +-
docs/impala.ditamap | 3 ++
docs/topics/impala_complex_types.xml | 34 +++++++------
...size.xml => impala_compute_processing_cost.xml} | 27 ++++++-----
docs/topics/impala_default_file_format.xml | 8 +++-
docs/topics/impala_expand_complex_types.xml | 10 ++--
docs/topics/impala_iceberg.xml | 2 +-
docs/topics/impala_incompatible_changes.xml | 20 ++++++++
... => impala_max_fragment_instances_per_node.xml} | 30 ++++++------
....xml => impala_processing_cost_min_threads.xml} | 26 +++++-----
.../org/apache/impala/analysis/SelectStmt.java | 11 +++--
.../java/org/apache/impala/analysis/TableDef.java | 17 ++++---
.../apache/impala/service/CatalogOpExecutor.java | 4 +-
.../org/apache/impala/analysis/AnalyzeDDLTest.java | 17 +++----
.../apache/impala/analysis/AnalyzeStmtsTest.java | 45 ++++++++++++-----
.../org/apache/impala/analysis/AnalyzerTest.java | 4 +-
.../apache/impala/analysis/ExprRewriterTest.java | 4 +-
.../java/org/apache/impala/analysis/ToSqlTest.java | 38 +++++++--------
.../org/apache/impala/planner/PlannerTest.java | 15 +++++-
testdata/bin/load_nested.py | 12 +++--
.../functional/functional_schema_template.sql | 12 +++++
.../queries/PlannerTest/ddl.test | 22 ++++-----
.../queries/PlannerTest/empty.test | 2 +-
.../queries/PlannerTest/resource-requirements.test | 4 +-
.../queries/PlannerTest/tpcds_cpu_cost/ddl.test | 22 ++++-----
.../queries/PlannerTest/tpcds_tuple_cache/ddl.test | 22 ++++-----
.../queries/QueryTest/acid-compute-stats.test | 6 +--
.../QueryTest/alter-table-hdfs-caching.test | 8 ++--
.../queries/QueryTest/alter-table.test | 10 ++--
.../QueryTest/compute-stats-incremental.test | 20 ++++----
.../queries/QueryTest/compute-stats.test | 4 +-
.../queries/QueryTest/corrupt-stats.test | 4 +-
.../queries/QueryTest/create-table-like-table.test | 2 +-
.../queries/QueryTest/grant_revoke.test | 3 +-
.../queries/QueryTest/hdfs-caching.test | 10 ++--
.../queries/QueryTest/iceberg-metadata-tables.test | 1 +
...migrated-table-field-id-resolution-complex.test | 8 ++++
.../queries/QueryTest/iceberg-query.test | 2 +-
.../queries/QueryTest/iceberg-v3-negative.test | 1 +
.../queries/QueryTest/insert_null.test | 2 +
.../queries/QueryTest/local-filesystem.test | 1 +
.../queries/QueryTest/max-nesting-depth.test | 12 +++++
.../QueryTest/nested-struct-in-select-list.test | 3 ++
.../QueryTest/nested-types-star-expansion.test | 4 ++
.../queries/QueryTest/nested-types-subplan.test | 4 ++
.../QueryTest/nested-types-tpch-mem-limit.test | 2 +
.../queries/QueryTest/paimon-query.test | 4 ++
.../queries/QueryTest/partition-col-types.test | 6 ++-
.../QueryTest/partition-ddl-predicates-all-fs.test | 12 ++---
.../ranger_column_masking_complex_types.test | 56 +++++++++++-----------
.../queries/QueryTest/ranger_row_filtering.test | 3 ++
.../functional-query/queries/QueryTest/set.test | 6 +--
.../queries/QueryTest/show-create-table.test | 8 ++--
.../QueryTest/union-const-scalar-expr-codegen.test | 2 +-
.../queries/QueryTest/unsupported-writers.test | 4 +-
.../virtual-column-file-position-parquet.test | 8 ++--
tests/comparison/db_connection.py | 2 +-
tests/custom_cluster/test_executor_groups.py | 6 ++-
tests/custom_cluster/test_lineage.py | 4 ++
.../test_metadata_no_events_processing.py | 5 +-
tests/custom_cluster/test_tuple_cache.py | 3 +-
tests/data_errors/test_data_errors.py | 2 +-
tests/metadata/test_compute_stats.py | 2 +-
tests/metadata/test_ddl.py | 15 +++---
tests/metadata/test_explain.py | 7 +--
tests/metadata/test_recover_partitions.py | 25 ++++++----
tests/query_test/test_compressed_formats.py | 2 +-
tests/query_test/test_delimited_text.py | 2 +-
tests/query_test/test_hdfs_caching.py | 5 +-
tests/query_test/test_insert_parquet.py | 11 +++--
tests/query_test/test_kudu.py | 6 ++-
tests/query_test/test_scanners.py | 7 ++-
tests/shell/test_shell_interactive.py | 2 +-
73 files changed, 455 insertions(+), 282 deletions(-)
copy docs/topics/{impala_compute_stats_min_sample_size.xml => impala_compute_processing_cost.xml} (56%)
copy docs/topics/{impala_compute_stats_min_sample_size.xml => impala_max_fragment_instances_per_node.xml} (54%)
copy docs/topics/{impala_compute_stats_min_sample_size.xml => impala_processing_cost_min_threads.xml} (60%)
(impala) 01/03: IMPALA-14156: [DOCS] Documentation for query options
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 407be0e24e0ac1422f762f790de378e15b01cc7a
Author: m-sanjana19 <ma...@gmail.com>
AuthorDate: Mon Jun 22 22:30:32 2026 +0530
IMPALA-14156: [DOCS] Documentation for query options
Adds documentation for the following query options, following the
existing descriptions in ImpalaService.thrift:
- COMPUTE_PROCESSING_COST
- PROCESSING_COST_MIN_THREADS
- MAX_FRAGMENT_INSTANCES_PER_NODE
Change-Id: I435c5eb65d049558eb6ec3fea1bc1572cba0669d
Reviewed-on: http://gerrit.cloudera.org:8080/24492
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Peter Rozsa <pr...@cloudera.com>
Reviewed-by: Noemi Pap-Takacs <np...@cloudera.com>
---
docs/impala.ditamap | 3 ++
docs/topics/impala_compute_processing_cost.xml | 48 ++++++++++++++++++++++
.../impala_max_fragment_instances_per_node.xml | 47 +++++++++++++++++++++
docs/topics/impala_processing_cost_min_threads.xml | 47 +++++++++++++++++++++
4 files changed, 145 insertions(+)
diff --git a/docs/impala.ditamap b/docs/impala.ditamap
index ea9c1b15d..5e6a54685 100644
--- a/docs/impala.ditamap
+++ b/docs/impala.ditamap
@@ -193,6 +193,7 @@ under the License.
<topicref rev="2.10.0 IMPALA-3200" href="topics/impala_buffer_pool_limit.xml"/>
<topicref href="topics/impala_compression_codec.xml"/>
<topicref href="topics/impala_compute_stats_min_sample_size.xml"/>
+ <topicref href="topics/impala_compute_processing_cost.xml"/>
<topicref href="topics/impala_debug_action.xml"/>
<topicref rev="2.9.0" href="topics/impala_decimal_v2.xml"/>
<topicref rev="3.2.0" href="topics/impala_default_file_format.xml"/>
@@ -222,6 +223,7 @@ under the License.
<topicref href="topics/impala_live_progress.xml"/>
<topicref href="topics/impala_live_summary.xml"/>
<topicref href="topics/impala_max_errors.xml"/>
+ <topicref href="topics/impala_max_fragment_instances_per_node.xml"/>
<topicref rev="3.1 IMPALA-6847" href="topics/impala_max_mem_estimate_for_admission.xml"/>
<topicref rev="2.5.0" href="topics/impala_max_num_runtime_filters.xml"/>
<topicref href="topics/impala_max_result_spooling_mem.xml"/>
@@ -247,6 +249,7 @@ under the License.
<topicref href="topics/impala_parquet_read_page_index.xml"/>
<topicref href="topics/impala_parquet_write_page_index.xml"/>
<topicref rev="2.6.0 IMPALA-3286" href="topics/impala_prefetch_mode.xml"/>
+ <topicref rev="2.6.0 IMPALA-3286" href="topics/impala_processing_cost_min_threads.xml"/>
<topicref href="topics/impala_query_timeout_s.xml"/>
<topicref href="topics/impala_refresh_updated_hms.xml"/>
<topicref rev="2.7.0" href="topics/impala_replica_preference.xml"/>
diff --git a/docs/topics/impala_compute_processing_cost.xml b/docs/topics/impala_compute_processing_cost.xml
new file mode 100644
index 000000000..8542165c5
--- /dev/null
+++ b/docs/topics/impala_compute_processing_cost.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="impala_compute_processing_cost" rev="5.0.0">
+ <title>COMPUTE_PROCESSING_COST Query Option</title>
+ <titlealts audience="PDF">
+ <navtitle>COMPUTE PROCESSING COST</navtitle>
+ </titlealts>
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="Impala Query Options"/>
+ <data name="Category" value="Performance"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Data Analysts"/>
+ </metadata>
+ </prolog>
+ <conbody>
+ <p>The <codeph>COMPUTE_PROCESSING_COST</codeph> query option determines whether the query
+ planner evaluates the CPU processing cost of a query.</p>
+ <p>When you enable this query option, the query planner estimates the CPU processing cost of
+ each query fragment. The query planner uses this estimate to adjust the degree of parallelism,
+ which represents the number of fragment instances scheduled per executor node. This cost-based
+ parallelism adjustment improves resource utilization compared to using a fixed degree of
+ parallelism.</p>
+ <p conref="../shared/impala_common.xml#common/type_boolean"/>
+ <p conref="../shared/impala_common.xml#common/default_false"/>
+ <p><b>Added in</b>: <keyword keyref="impala43" id="IMPALA-11604"/></p>
+ <p conref="../shared/impala_common.xml#common/usage_notes_blurb"/>
+ </conbody>
+</concept>
diff --git a/docs/topics/impala_max_fragment_instances_per_node.xml b/docs/topics/impala_max_fragment_instances_per_node.xml
new file mode 100644
index 000000000..bff784019
--- /dev/null
+++ b/docs/topics/impala_max_fragment_instances_per_node.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="impala_max_fragment_instances_per_node">
+
+ <title>MAX_FRAGMENT_INSTANCES_PER_NODE Query Option</title>
+ <titlealts audience="PDF"><navtitle>MAX FRAGMENT INSTANCES PER NODE</navtitle></titlealts>
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="Impala Query Options"/>
+ <data name="Category" value="Performance"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Data Analysts"/>
+ </metadata>
+ </prolog>
+ <conbody>
+ <p>The <codeph>MAX_FRAGMENT_INSTANCES_PER_NODE</codeph> query option sets the maximum number of
+ fragment instances per node that the processing cost algorithm assigns to a query
+ fragment.</p>
+ <p>The recommended practice is to set this query option to a value lower than or equal to the
+ number of physical cores in an executor node. This query option is ignored if the selected
+ executor group has a lower <codeph>max-query-cpu-core-per-node-limit</codeph> configuration,
+ or if the <codeph>PROCESSING_COST_MIN_THREADS</codeph> query option has a higher value.</p>
+ <p conref="../shared/impala_common.xml#common/type_integer"/>
+ <p><b>Allowed values:</b> Integers in the range [1, 128].</p>
+ <p><b>Default:</b> 128</p>
+ <p><b>Added in:</b><keyword keyref="impala43" id="IMPALA-12091"/></p>
+ </conbody>
+</concept>
diff --git a/docs/topics/impala_processing_cost_min_threads.xml b/docs/topics/impala_processing_cost_min_threads.xml
new file mode 100644
index 000000000..7983f07d9
--- /dev/null
+++ b/docs/topics/impala_processing_cost_min_threads.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="impala_processing_cost_min_threads" rev="5.0.0">
+ <title>PROCESSING_COST_MIN_THREADS Query Option</title>
+ <titlealts audience="PDF">
+ <navtitle>PROCESSING COST MIN THREADS</navtitle>
+ </titlealts>
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="Impala Query Options"/>
+ <data name="Category" value="Performance"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Data Analysts"/>
+ </metadata>
+ </prolog>
+ <conbody>
+ <p>The <codeph>PROCESSING_COST_MIN_THREADS</codeph> query option sets the minimum number of
+ fragment instances per node that the processing cost algorithm assigns to a query
+ fragment.</p>
+ <p>The recommended practice is to set this query option to a value lower than or equal to the
+ number of physical cores in an executor node.</p>
+ <p conref="../shared/impala_common.xml#common/type_integer"/>
+ <p><b>Allowed values:</b> Integers in the range [1, 128].</p>
+ <p><b>Default:</b> 1</p>
+ <p><b>Added in:</b>
+ <keyword keyref="impala43" id="IMPALA-11604"/></p>
+ </conbody>
+</concept>
(impala) 02/03: IMPALA-2210: Make Parquet the default file format
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 848ef4735f49e9707456ffd0f0f2079f2e83414c
Author: Peter Rozsa <pr...@cloudera.com>
AuthorDate: Fri Jul 10 14:53:56 2026 +0200
IMPALA-2210: Make Parquet the default file format
Change the default value of the DEFAULT_FILE_FORMAT query option from
TEXT to PARQUET. This affects all CREATE TABLE statements that do not
specify an explicit STORED AS clause, including plain CREATE TABLE,
CREATE TABLE AS SELECT, CREATE TABLE LIKE PARQUET, and CREATE TABLE
with CLUSTERED BY.
Change-Id: I46a5d16dfd780aae41623ebfc9742dfbd2a596ea
Assisted-by: Claude Sonnet 4.5 (OpenCode)
Reviewed-on: http://gerrit.cloudera.org:8080/24569
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
---
common/thrift/Query.thrift | 2 +-
docs/topics/impala_default_file_format.xml | 8 ++++-
.../java/org/apache/impala/analysis/TableDef.java | 17 ++++++----
.../apache/impala/service/CatalogOpExecutor.java | 4 +--
.../org/apache/impala/analysis/AnalyzeDDLTest.java | 17 +++++-----
.../apache/impala/analysis/ExprRewriterTest.java | 4 +--
.../java/org/apache/impala/analysis/ToSqlTest.java | 38 +++++++++++-----------
testdata/bin/load_nested.py | 12 ++++---
.../functional/functional_schema_template.sql | 12 +++++++
.../queries/PlannerTest/ddl.test | 22 ++++++-------
.../queries/PlannerTest/resource-requirements.test | 4 +--
.../queries/PlannerTest/tpcds_cpu_cost/ddl.test | 22 ++++++-------
.../queries/PlannerTest/tpcds_tuple_cache/ddl.test | 22 ++++++-------
.../queries/QueryTest/acid-compute-stats.test | 6 ++--
.../QueryTest/alter-table-hdfs-caching.test | 8 ++---
.../queries/QueryTest/alter-table.test | 10 +++---
.../QueryTest/compute-stats-incremental.test | 20 +++++++-----
.../queries/QueryTest/compute-stats.test | 4 +--
.../queries/QueryTest/corrupt-stats.test | 4 +--
.../queries/QueryTest/create-table-like-table.test | 2 +-
.../queries/QueryTest/grant_revoke.test | 3 +-
.../queries/QueryTest/hdfs-caching.test | 10 +++---
.../queries/QueryTest/insert_null.test | 2 ++
.../queries/QueryTest/local-filesystem.test | 1 +
.../queries/QueryTest/partition-col-types.test | 6 ++--
.../QueryTest/partition-ddl-predicates-all-fs.test | 12 +++----
.../functional-query/queries/QueryTest/set.test | 6 ++--
.../queries/QueryTest/show-create-table.test | 8 ++---
.../QueryTest/union-const-scalar-expr-codegen.test | 2 +-
.../queries/QueryTest/unsupported-writers.test | 4 +--
tests/comparison/db_connection.py | 2 +-
tests/custom_cluster/test_executor_groups.py | 6 +++-
.../test_metadata_no_events_processing.py | 5 +--
tests/custom_cluster/test_tuple_cache.py | 3 +-
tests/data_errors/test_data_errors.py | 2 +-
tests/metadata/test_compute_stats.py | 2 +-
tests/metadata/test_ddl.py | 15 +++++----
tests/metadata/test_explain.py | 7 ++--
tests/metadata/test_recover_partitions.py | 25 +++++++++-----
tests/query_test/test_compressed_formats.py | 2 +-
tests/query_test/test_delimited_text.py | 2 +-
tests/query_test/test_hdfs_caching.py | 5 +--
tests/query_test/test_insert_parquet.py | 11 ++++---
tests/query_test/test_scanners.py | 7 ++--
tests/shell/test_shell_interactive.py | 2 +-
45 files changed, 224 insertions(+), 164 deletions(-)
diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift
index 15f34babf..8c76e36aa 100644
--- a/common/thrift/Query.thrift
+++ b/common/thrift/Query.thrift
@@ -422,7 +422,7 @@ struct TQueryOptions {
// See comment in ImpalaService.thrift.
79: optional CatalogObjects.THdfsFileFormat default_file_format =
- CatalogObjects.THdfsFileFormat.TEXT;
+ CatalogObjects.THdfsFileFormat.PARQUET;
// See comment in ImpalaService.thrift.
80: optional TParquetTimestampType parquet_timestamp_type =
diff --git a/docs/topics/impala_default_file_format.xml b/docs/topics/impala_default_file_format.xml
index 7d41cc6ae..14cdaf0c8 100644
--- a/docs/topics/impala_default_file_format.xml
+++ b/docs/topics/impala_default_file_format.xml
@@ -85,9 +85,15 @@ under the License.
</p>
<p>
- <b>Default:</b> <codeph>TEXT</codeph> (<codeph>0</codeph>)
+ <b>Default:</b> <codeph>PARQUET</codeph> (<codeph>4</codeph>)
</p>
+ <note>
+ The default value was changed to <codeph>PARQUET</codeph> starting from
+ <keyword keyref="impala50"/>. In earlier releases the default was
+ <codeph>TEXT</codeph>.
+ </note>
+
<p>
<b>Added in:</b> <keyword keyref="impala32"/>
</p>
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableDef.java b/fe/src/main/java/org/apache/impala/analysis/TableDef.java
index 322c7ed8a..6b5c990de 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableDef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableDef.java
@@ -176,9 +176,13 @@ class TableDef {
Preconditions.checkNotNull(serdeProperties);
this.serdeProperties = serdeProperties;
// The file format passed via STORED AS <file format> has a higher precedence than
- // the one set in query options.
- this.fileFormat = (fileFormat != null) ?
- fileFormat : queryOptions.getDefault_file_format();
+ // the one set in query options. When STORED AS is absent but a ROW FORMAT clause is
+ // present, the file format is implicitly TEXT for backward compatibility, since
+ // ROW FORMAT only applies to TEXT tables. Otherwise, fall back to the
+ // DEFAULT_FILE_FORMAT query option.
+ this.fileFormat = (fileFormat != null) ? fileFormat
+ : (rowFormat != null) ? THdfsFileFormat.TEXT
+ : queryOptions.getDefault_file_format();
this.location = location;
this.cachingOp = cachingOp;
Preconditions.checkNotNull(tblProperties);
@@ -187,10 +191,11 @@ class TableDef {
}
public Options(String comment, TQueryOptions queryOptions) {
- // Passing null to file format so that it uses the file format from the query option
- // if specified, otherwise it will use the default file format, which is TEXT.
+ // Passing null to row format and file format so that the file format is taken from
+ // the query option if specified, otherwise the default file format, which is
+ // PARQUET. There is no ROW FORMAT clause on this code path, hence null row format.
this(null, new Pair<>(ImmutableList.of(), TSortingOrder.LEXICAL), comment,
- RowFormat.DEFAULT_ROW_FORMAT, new HashMap<>(), /* file format */null, null,
+ /* row format */null, new HashMap<>(), /* file format */null, null,
null, new HashMap<>(), queryOptions);
}
}
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index a30a27f2e..14257ba29 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -4639,8 +4639,8 @@ public class CatalogOpExecutor {
} else if (srcTable instanceof View) {
// Here, source table is a view which has no input format. So to be
// consistent with CREATE TABLE, default input format is assumed to be
- // TEXT unless otherwise specified.
- setStorageDescriptorFileFormat(tbl.getSd(), THdfsFileFormat.TEXT);
+ // PARQUET unless otherwise specified.
+ setStorageDescriptorFileFormat(tbl.getSd(), THdfsFileFormat.PARQUET);
}
// Set the row count of this table to unknown.
tbl.putToParameters(StatsSetupConst.ROW_COUNT, "-1");
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index aa532d593..3b5b89faf 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -1141,7 +1141,7 @@ public class AnalyzeDDLTest extends FrontendTestBase {
String tmpTableName =
QueryStringBuilder.createTmpTableName("functional", "tmp_table");
AnalyzesOk("create table " + tmpTableName + " (id int) with serdeproperties(" +
- "'serialization.encoding'='GBK')");
+ "'serialization.encoding'='GBK') stored as textfile");
String [] unsupportedFileFormatDbs =
{"functional_parquet", "functional_rc", "functional_avro"};
@@ -1182,11 +1182,11 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"'serialization.encoding'='NonexistentEncoding')",
"Unsupported encoding: NonexistentEncoding.");
AnalysisError("create table " + tmpTableName + " (id int) with serdeproperties(" +
- "'serialization.encoding'='UTF-16')",
+ "'serialization.encoding'='UTF-16') stored as textfile",
"Property 'serialization.encoding' only supports encodings in which line " +
"delimiter is compatible with ASCII.");
AnalysisError("create table " + tmpTableName + " (id int) with serdeproperties(" +
- "'serialization.encoding'='NonexistentEncoding')",
+ "'serialization.encoding'='NonexistentEncoding') stored as textfile",
"Unsupported encoding: NonexistentEncoding.");
// Numeric table stats properties validation
@@ -3115,20 +3115,21 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// IMPALA-2251: it should not be possible to create text tables with the same
// delimiter character used for multiple purposes.
AnalysisError("create table functional.broken_text_table (c int) " +
- "row format delimited fields terminated by '\001' lines terminated by '\001'",
+ "row format delimited fields terminated by '\001' lines terminated by '\001' " +
+ "stored as textfile",
"Field delimiter and line delimiter have same value: byte 1");
AnalysisError("create table functional.broken_text_table (c int) " +
- "row format delimited lines terminated by '\001'",
+ "row format delimited lines terminated by '\001' stored as textfile",
"Field delimiter and line delimiter have same value: byte 1");
AnalysisError("create table functional.broken_text_table (c int) " +
- "row format delimited fields terminated by '\012'",
+ "row format delimited fields terminated by '\012' stored as textfile",
"Field delimiter and line delimiter have same value: byte 10");
AnalyzesOk("create table functional.broken_text_table (c int) " +
- "row format delimited escaped by '\001'",
+ "row format delimited escaped by '\001' stored as textfile",
"Field delimiter and escape character have same value: byte 1. " +
"Escape character will be ignored");
AnalyzesOk("create table functional.broken_text_table (c int) " +
- "row format delimited escaped by 'x' lines terminated by 'x'",
+ "row format delimited escaped by 'x' lines terminated by 'x' stored as textfile",
"Line delimiter and escape character have same value: byte 120. " +
"Escape character will be ignored");
diff --git a/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java b/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
index 92ae7f919..204af4d96 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
@@ -333,10 +333,10 @@ public class ExprRewriterTest extends AnalyzerTest {
assertToSql(ctx,
"create table ctas_test as select 1 + 1",
"CREATE TABLE default.ctas_test\n" +
- "STORED AS TEXTFILE\n" +
+ "STORED AS PARQUET\n" +
" AS SELECT 1 + 1",
"CREATE TABLE default.ctas_test\n" +
- "STORED AS TEXTFILE\n" +
+ "STORED AS PARQUET\n" +
" AS SELECT 2");
//--------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
index 64bf42f56..52f46d934 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
@@ -327,29 +327,29 @@ public class ToSqlTest extends FrontendTestBase {
"comment 'This is a test'",
"default",
"CREATE TABLE default.p ( a INT ) PARTITIONED BY ( day STRING ) " +
- "SORT BY LEXICAL ( a ) COMMENT 'This is a test' STORED AS TEXTFILE" , true);
+ "SORT BY LEXICAL ( a ) COMMENT 'This is a test' STORED AS PARQUET" , true);
testToSql("create table p (a int, b int) partitioned by (day string) sort by (a ,b) ",
"default",
"CREATE TABLE default.p ( a INT, b INT ) PARTITIONED BY ( day STRING ) " +
- "SORT BY LEXICAL ( a, b ) STORED AS TEXTFILE" , true);
+ "SORT BY LEXICAL ( a, b ) STORED AS PARQUET" , true);
// Table with SORT BY LEXICAL clause.
testToSql("create table p (a int) partitioned by (day string) sort by lexical (a) " +
"comment 'This is a test'",
"default",
"CREATE TABLE default.p ( a INT ) PARTITIONED BY ( day STRING ) " +
- "SORT BY LEXICAL ( a ) COMMENT 'This is a test' STORED AS TEXTFILE" , true);
+ "SORT BY LEXICAL ( a ) COMMENT 'This is a test' STORED AS PARQUET" , true);
testToSql("create table p (a int, b int) partitioned by (day string) sort by " +
"lexical (a, b)",
"default",
"CREATE TABLE default.p ( a INT, b INT ) PARTITIONED BY ( day STRING ) " +
- "SORT BY LEXICAL ( a, b ) STORED AS TEXTFILE" , true);
+ "SORT BY LEXICAL ( a, b ) STORED AS PARQUET" , true);
// Table with SORT BY ZORDER clause.
testToSql("create table p (a int, b int) partitioned by (day string) sort by zorder" +
"(a ,b) ", "default",
"CREATE TABLE default.p ( a INT, b INT ) PARTITIONED BY ( day STRING ) " +
- "SORT BY ZORDER ( a, b ) STORED AS TEXTFILE" , true);
+ "SORT BY ZORDER ( a, b ) STORED AS PARQUET" , true);
// Kudu table with a TIMESTAMP column default value
String kuduMasters = "127.0.0.1";
@@ -369,7 +369,7 @@ public class ToSqlTest extends FrontendTestBase {
// TODO: Add support for displaying constraint information (DISABLE, NOVALIDATE, RELY)
testToSql("create table pk(id int, year string, primary key (id, year))", "default",
"CREATE TABLE default.pk ( id INT, year STRING, PRIMARY KEY (id, year) ) "
- + "STORED AS TEXTFILE", true);
+ + "STORED AS PARQUET", true);
// Foreign Key test requires a valid primary key table.
addTestDb("test_pk_fk", "Test DB for PK/FK tests");
@@ -380,7 +380,7 @@ public class ToSqlTest extends FrontendTestBase {
+ "KEY (a) REFERENCES functional.parent_table_2(a))", "CREATE TABLE "
+ "test_pk_fk.fk ( seq INT, id INT, year STRING, a INT, FOREIGN KEY(id, year) "
+ "REFERENCES functional.parent_table(id, year), FOREIGN KEY(a) REFERENCES "
- + "functional.parent_table_2(a) ) STORED AS TEXTFILE", true);
+ + "functional.parent_table_2(a) ) STORED AS PARQUET", true);
// Test create table for paimon table.
testToSql("CREATE TABLE test_create_managed_paimon_table ("
+ "user_id BIGINT, item_id BIGINT, behavior STRING) STORED AS PAIMON",
@@ -431,32 +431,32 @@ public class ToSqlTest extends FrontendTestBase {
testToSql("create table p partitioned by (int_col) as " +
"select double_col, int_col from functional.alltypes", "default",
"CREATE TABLE default.p PARTITIONED BY ( int_col ) STORED AS " +
- "TEXTFILE AS SELECT double_col, int_col FROM functional.alltypes",
+ "PARQUET AS SELECT double_col, int_col FROM functional.alltypes",
true);
// Table with a comment.
testToSql("create table p partitioned by (int_col) comment 'This is a test' as " +
"select double_col, int_col from functional.alltypes", "default",
"CREATE TABLE default.p PARTITIONED BY ( int_col ) COMMENT 'This is a test' " +
- "STORED AS TEXTFILE AS SELECT double_col, int_col FROM functional.alltypes",
+ "STORED AS PARQUET AS SELECT double_col, int_col FROM functional.alltypes",
true);
// Table with SORT BY clause.
testToSql("create table p partitioned by (int_col) sort by (string_col) as " +
"select double_col, string_col, int_col from functional.alltypes", "default",
"CREATE TABLE default.p PARTITIONED BY ( int_col ) SORT BY LEXICAL " +
- "( string_col ) STORED AS TEXTFILE AS SELECT double_col, string_col, int_col " +
+ "( string_col ) STORED AS PARQUET AS SELECT double_col, string_col, int_col " +
"FROM functional.alltypes", true);
// Table with SORT BY LEXICAL clause.
testToSql("create table p partitioned by (int_col) sort by lexical (string_col) as " +
"select double_col, string_col, int_col from functional.alltypes", "default",
"CREATE TABLE default.p PARTITIONED BY ( int_col ) SORT BY LEXICAL " +
- "( string_col ) STORED AS TEXTFILE AS SELECT double_col, string_col, int_col " +
+ "( string_col ) STORED AS PARQUET AS SELECT double_col, string_col, int_col " +
"FROM functional.alltypes", true);
// Table with SORT BY ZORDER clause.
testToSql("create table p partitioned by (string_col) sort by zorder (int_col, " +
"bool_col) as select int_col, bool_col, string_col from functional.alltypes",
"default",
"CREATE TABLE default.p PARTITIONED BY ( string_col ) SORT BY ZORDER " +
- "( int_col, bool_col ) STORED AS TEXTFILE AS SELECT " +
+ "( int_col, bool_col ) STORED AS PARQUET AS SELECT " +
"int_col, bool_col, string_col FROM functional.alltypes", true);
// Kudu table with multiple partition params
String kuduMasters = "127.0.0.1";
@@ -495,26 +495,26 @@ public class ToSqlTest extends FrontendTestBase {
"'/test-warehouse/schemas/alltypestiny.parquet'", "default",
"CREATE TABLE IF NOT EXISTS default.p LIKE PARQUET " +
"'hdfs://localhost:20500/test-warehouse/schemas/alltypestiny.parquet' " +
- "STORED AS TEXTFILE", true);
+ "STORED AS PARQUET", true);
// Table with sort columns.
testToSql("create table if not exists p like parquet " +
"'/test-warehouse/schemas/alltypestiny.parquet' sort by (int_col, id)", "default",
"CREATE TABLE IF NOT EXISTS default.p LIKE PARQUET " +
"'hdfs://localhost:20500/test-warehouse/schemas/alltypestiny.parquet' " +
- "SORT BY LEXICAL ( int_col, id ) STORED AS TEXTFILE", true);
+ "SORT BY LEXICAL ( int_col, id ) STORED AS PARQUET", true);
// Table with sort LEXICAL columns.
testToSql("create table if not exists p like parquet " +
"'/test-warehouse/schemas/alltypestiny.parquet' sort by lexical (int_col, id)",
"default",
"CREATE TABLE IF NOT EXISTS default.p LIKE PARQUET " +
"'hdfs://localhost:20500/test-warehouse/schemas/alltypestiny.parquet' " +
- "SORT BY LEXICAL ( int_col, id ) STORED AS TEXTFILE", true);
+ "SORT BY LEXICAL ( int_col, id ) STORED AS PARQUET", true);
// Table with ZORDER sort columns.
testToSql("create table if not exists p like parquet " +
"'/test-warehouse/schemas/alltypestiny.parquet' sort by zorder (int_col, id)",
"default", "CREATE TABLE IF NOT EXISTS default.p LIKE PARQUET " +
"'hdfs://localhost:20500/test-warehouse/schemas/alltypestiny.parquet' " +
- "SORT BY ZORDER ( int_col, id ) STORED AS TEXTFILE", true);
+ "SORT BY ZORDER ( int_col, id ) STORED AS PARQUET", true);
}
@Test
@@ -522,17 +522,17 @@ public class ToSqlTest extends FrontendTestBase {
testToSql("create table bucketed_table ( a int ) " +
"CLUSTERED BY ( a ) into 24 buckets", "default",
"CREATE TABLE default.bucketed_table ( a INT ) " +
- "CLUSTERED BY ( a ) INTO 24 BUCKETS STORED AS TEXTFILE", true);
+ "CLUSTERED BY ( a ) INTO 24 BUCKETS STORED AS PARQUET", true);
testToSql("create table bucketed_table1 ( a int ) " +
"partitioned by ( dt string ) CLUSTERED BY ( a ) into 24 buckets", "default",
"CREATE TABLE default.bucketed_table1 ( a INT ) " +
"PARTITIONED BY ( dt STRING ) CLUSTERED BY ( a ) INTO 24 BUCKETS " +
- "STORED AS TEXTFILE", true);
+ "STORED AS PARQUET", true);
testToSql("create table bucketed_table2 ( a int, s string ) partitioned " +
"by ( dt string ) CLUSTERED BY ( a ) sort by (s) into 24 buckets",
"default", "CREATE TABLE default.bucketed_table2 " +
"( a INT, s STRING ) PARTITIONED BY ( dt STRING ) CLUSTERED BY ( a ) SORT BY " +
- "LEXICAL ( s ) INTO 24 BUCKETS STORED AS TEXTFILE", true);
+ "LEXICAL ( s ) INTO 24 BUCKETS STORED AS PARQUET", true);
}
@Test
diff --git a/testdata/bin/load_nested.py b/testdata/bin/load_nested.py
index 9cba010af..2def9db42 100755
--- a/testdata/bin/load_nested.py
+++ b/testdata/bin/load_nested.py
@@ -139,7 +139,8 @@ def load():
LOG.info("Creating temp orders (chunk {chunk} of {chunks})".format(
chunk=(chunk_idx + 1), chunks=chunks))
if chunk_idx == 0:
- impala.execute("CREATE TABLE tmp_orders_string AS " + tmp_orders_sql)
+ impala.execute(
+ "CREATE TABLE tmp_orders_string STORED AS TEXTFILE AS " + tmp_orders_sql)
else:
impala.execute("INSERT INTO TABLE tmp_orders_string " + tmp_orders_sql)
@@ -169,7 +170,8 @@ def load():
LOG.info("Creating temp customers (chunk {chunk} of {chunks})".format(
chunk=(chunk_idx + 1), chunks=chunks))
if chunk_idx == 0:
- impala.execute("CREATE TABLE tmp_customer_string AS " + tmp_customer_sql)
+ impala.execute(
+ "CREATE TABLE tmp_customer_string STORED AS TEXTFILE AS " + tmp_customer_sql)
else:
impala.execute("INSERT INTO TABLE tmp_customer_string " + tmp_customer_sql)
@@ -220,6 +222,7 @@ def load():
LOG.info("Creating temp regions")
impala.execute(r"""
CREATE TABLE tmp_region_string
+ STORED AS TEXTFILE
AS SELECT
r_regionkey, r_name, r_comment,
GROUP_CONCAT(
@@ -249,8 +252,9 @@ def load():
# avoid duplicated data.
LOG.info("Creating temp suppliers")
impala.execute(r"""
- CREATE TABLE tmp_supplier_string AS
- SELECT
+ CREATE TABLE tmp_supplier_string
+ STORED AS TEXTFILE
+ AS SELECT
s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment,
GROUP_CONCAT(
CONCAT(
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index fdde1b3b4..96dbf1a8d 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -2387,6 +2387,7 @@ parent_table
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
id INT, year string, primary key(id, year) DISABLE NOVALIDATE RELY)
row format delimited fields terminated by ','
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- ROW_FORMAT
delimited fields terminated by '',''
@@ -2402,6 +2403,7 @@ parent_table_2
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
a INT, primary key(a) DISABLE NOVALIDATE RELY)
row format delimited fields terminated by ','
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- ROW_FORMAT
delimited fields terminated by ','
@@ -2420,6 +2422,7 @@ seq int, id int, year string, a int, primary key(seq) DISABLE NOVALIDATE RELY, f
RELY, foreign key(a) references {db_name}{db_suffix}.parent_table_2(a) DISABLE
NOVALIDATE RELY)
row format delimited fields terminated by ','
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- ROW_FORMAT
delimited fields terminated by ','
@@ -2544,6 +2547,7 @@ table_no_newline
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
id INT, col_1 BOOLEAN, col_2 DOUBLE, col_3 TIMESTAMP)
row format delimited fields terminated by ','
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
LOAD DATA LOCAL INPATH '{impala_home}/testdata/data/table_no_newline.csv'
@@ -2558,6 +2562,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
id INT, col_1 BOOLEAN, col_2 DOUBLE, col_3 TIMESTAMP)
partitioned by (year INT, month INT)
row format delimited fields terminated by ','
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
ALTER TABLE {db_name}{db_suffix}.{table_name} ADD IF NOT EXISTS PARTITION (year=2015, month=3);
ALTER TABLE {db_name}{db_suffix}.{table_name} ADD IF NOT EXISTS PARTITION (year=2010, month=3);
@@ -2684,6 +2689,7 @@ testescape_16_lf
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
col string)
row format delimited fields terminated by ',' escaped by '\\'
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
`${IMPALA_HOME}/testdata/common/text_delims_table.py --table_dir '/tmp/testescape_16_lf' --file_len 16 --only_newline
@@ -2696,6 +2702,7 @@ testescape_16_crlf
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
col string)
row format delimited fields terminated by ',' escaped by '\\'
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
`${IMPALA_HOME}/testdata/common/text_delims_table.py --table_dir '/tmp/testescape_16_crlf' --file_len 16
@@ -2708,6 +2715,7 @@ testescape_17_lf
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
col string)
row format delimited fields terminated by ',' escaped by '\\'
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
`${IMPALA_HOME}/testdata/common/text_delims_table.py --table_dir '/tmp/testescape_17_lf' --file_len 17 --only_newline
@@ -2720,6 +2728,7 @@ testescape_17_crlf
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
col string)
row format delimited fields terminated by ',' escaped by '\\'
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
`${IMPALA_HOME}/testdata/common/text_delims_table.py --table_dir '/tmp/testescape_17_crlf' --file_len 17
@@ -2732,6 +2741,7 @@ testescape_32_lf
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
col string)
row format delimited fields terminated by ',' escaped by '\\'
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
`${IMPALA_HOME}/testdata/common/text_delims_table.py --table_dir '/tmp/testescape_32_lf' --file_len 32 --only_newline
@@ -2744,6 +2754,7 @@ testescape_32_crlf
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
col string)
row format delimited fields terminated by ',' escaped by '\\'
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
`${IMPALA_HOME}/testdata/common/text_delims_table.py --table_dir '/tmp/testescape_32_crlf' --file_len 32
@@ -2756,6 +2767,7 @@ alltimezones
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
timezone STRING, utctime TIMESTAMP, localtime TIMESTAMP)
row format delimited fields terminated by ','
+STORED AS TEXTFILE
LOCATION '/test-warehouse/{table_name}';
---- LOAD
LOAD DATA LOCAL INPATH '{impala_home}/testdata/data/timezoneverification.csv'
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/ddl.test b/testdata/workloads/functional-planner/queries/PlannerTest/ddl.test
index 84243c01f..29b378ff1 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/ddl.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/ddl.test
@@ -1,4 +1,4 @@
-create table t as select * from functional.alltypes
+create table t stored as textfile as select * from functional.alltypes
---- PLAN
WRITE TO HDFS [default.t, OVERWRITE=false]
| partitions=1
@@ -14,7 +14,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false]
partitions=24/24 files=24 size=478.45KB
row-size=89B cardinality=7.30K
====
-create table t as
+create table t stored as textfile as
select distinct a.int_col, a.string_col from functional.alltypes a
inner join functional.alltypessmall b on (a.id = b.id)
where a.year = 2009 and b.month = 2
@@ -74,7 +74,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false]
row-size=21B cardinality=3.65K
====
# CTAS with a view that has a limit clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select int_col from functional.hive_view
---- PLAN
WRITE TO HDFS [default.t, OVERWRITE=false]
@@ -97,7 +97,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false]
row-size=0B cardinality=1
====
# CTAS with multiple nested inline views that have a limit clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from
(select * from
(select int_col from functional.alltypestiny limit 1) v1 ) v2
@@ -123,7 +123,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false]
====
# CTAS with a select statement that has a limit and offset clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from functional.alltypestiny order by id limit 1 offset 5
---- PLAN
WRITE TO HDFS [default.t, OVERWRITE=false]
@@ -154,7 +154,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false]
row-size=89B cardinality=8
====
# CTAS with an inline view that has a limit and offset clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from
(select int_col from functional.alltypestiny order by id limit 2 offset 5) v
---- PLAN
@@ -186,7 +186,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false]
row-size=8B cardinality=8
====
# CTAS with sort columns
-create table t sort by (int_col, bool_col) as
+create table t sort by (int_col, bool_col) stored as textfile as
select * from functional.alltypes
---- PLAN
WRITE TO HDFS [default.t, OVERWRITE=false]
@@ -212,7 +212,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false]
row-size=89B cardinality=7.30K
====
# CTAS with partitions and sort columns
-create table t partitioned by (year, month) sort by (int_col, bool_col) as
+create table t partitioned by (year, month) sort by (int_col, bool_col) stored as textfile as
select * from functional.alltypes
---- PLAN
WRITE TO HDFS [default.t, OVERWRITE=false, PARTITION-KEYS=(year,month)]
@@ -245,7 +245,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false, PARTITION-KEYS=(year,month)]
# noshuffle hint would lead to a different plan.
# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
# because IMPALA-5293 changed clustered to be the default on 3.x.
-create /*+ noclustered */table t partitioned by (year, month) as
+create /*+ noclustered */table t partitioned by (year, month) stored as textfile as
select * from functional.alltypes
---- DISTRIBUTEDPLAN
WRITE TO HDFS [default.t, OVERWRITE=false, PARTITION-KEYS=(functional.alltypes.year,functional.alltypes.month)]
@@ -262,7 +262,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false, PARTITION-KEYS=(functional.alltypes.y
# in insert.test.
# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
# because IMPALA-5293 changed clustered to be the default on 3.x.
-create /*+ noshuffle, noclustered */ table t partitioned by (year, month) as
+create /*+ noshuffle, noclustered */ table t partitioned by (year, month) stored as textfile as
select * from functional.alltypes
---- DISTRIBUTEDPLAN
WRITE TO HDFS [default.t, OVERWRITE=false, PARTITION-KEYS=(functional.alltypes.year,functional.alltypes.month)]
@@ -273,7 +273,7 @@ WRITE TO HDFS [default.t, OVERWRITE=false, PARTITION-KEYS=(functional.alltypes.y
row-size=89B cardinality=7.30K
====
# CTAS with more complex select query
-create table t partitioned by (c_nationkey) sort by (c_custkey) as
+create table t partitioned by (c_nationkey) sort by (c_custkey) stored as textfile as
select c_custkey, max(o_totalprice) as maxprice, c_nationkey
from tpch.orders join tpch.customer on c_custkey = o_custkey
where c_nationkey < 10
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 0a66286c4..b7221b20d 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -1873,7 +1873,7 @@ WRITE TO HDFS [functional.alltypesnopart, OVERWRITE=false]
====
# Dynamically partitioned HDFS insert
create table dummy_insert
-partitioned by (l_partkey) as
+partitioned by (l_partkey) stored as textfile as
select l_comment, l_partkey from tpch.lineitem
---- PLAN
Max Per-Host Resource Reservation: Memory=20.00MB Threads=2
@@ -1973,7 +1973,7 @@ Per-Instance Resources: mem-estimate=89.18MB mem-reservation=8.00MB thread-reser
====
# Noclustered dynamically partitioned HDFS insert.
create /* +noclustered */ table dummy_insert
-partitioned by (l_partkey) as
+partitioned by (l_partkey) stored as textfile as
select l_comment, l_partkey from tpch.lineitem
---- PLAN
Max Per-Host Resource Reservation: Memory=8.00MB Threads=2
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_cpu_cost/ddl.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_cpu_cost/ddl.test
index c29ffab78..4288646d4 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_cpu_cost/ddl.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_cpu_cost/ddl.test
@@ -1,5 +1,5 @@
# Some of tests here are copied from ../ddl.test
-create table t as select * from functional.alltypes
+create table t stored as textfile as select * from functional.alltypes
---- PLAN
Max Per-Host Resource Reservation: Memory=32.00KB Threads=1
Per-Host Resource Estimates: Memory=16MB
@@ -51,7 +51,7 @@ max-parallelism=10 segment-costs=[62944]
tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
in pipelines: 00(GETNEXT)
====
-create table t as
+create table t stored as textfile as
select distinct a.int_col, a.string_col from functional.alltypes a
inner join functional.alltypessmall b on (a.id = b.id)
where a.year = 2009 and b.month = 2
@@ -187,7 +187,7 @@ max-parallelism=10 segment-costs=[9666, 20] cpu-comparison-result=11 [max(10 (se
in pipelines: 00(GETNEXT)
====
# CTAS with a view that has a limit clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select int_col from functional.hive_view
---- PLAN
Max Per-Host Resource Reservation: Memory=32.00KB Threads=1
@@ -244,7 +244,7 @@ max-parallelism=10 segment-costs=[17343]
in pipelines: 00(GETNEXT)
====
# CTAS with multiple nested inline views that have a limit clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from
(select * from
(select int_col from functional.alltypestiny limit 1) v1 ) v2
@@ -304,7 +304,7 @@ max-parallelism=4 segment-costs=[16]
====
# CTAS with a select statement that has a limit and offset clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from functional.alltypestiny order by id limit 1 offset 5
---- PLAN
Max Per-Host Resource Reservation: Memory=8.00KB Threads=1
@@ -373,7 +373,7 @@ max-parallelism=4 segment-costs=[29, 0]
in pipelines: 00(GETNEXT)
====
# CTAS with an inline view that has a limit and offset clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from
(select int_col from functional.alltypestiny order by id limit 2 offset 5) v
---- PLAN
@@ -443,7 +443,7 @@ max-parallelism=4 segment-costs=[29, 0]
in pipelines: 00(GETNEXT)
====
# CTAS with sort columns
-create table t sort by (int_col, bool_col) as
+create table t sort by (int_col, bool_col) stored as textfile as
select * from functional.alltypes
---- PLAN
Max Per-Host Resource Reservation: Memory=12.03MB Threads=1
@@ -509,7 +509,7 @@ max-parallelism=10 segment-costs=[62944]
in pipelines: 00(GETNEXT)
====
# CTAS with partitions and sort columns
-create table t partitioned by (year, month) sort by (int_col, bool_col) as
+create table t partitioned by (year, month) sort by (int_col, bool_col) stored as textfile as
select * from functional.alltypes
---- PLAN
Max Per-Host Resource Reservation: Memory=12.03MB Threads=1
@@ -580,7 +580,7 @@ max-parallelism=10 segment-costs=[62944]
# noshuffle hint would lead to a different plan.
# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
# because IMPALA-5293 changed clustered to be the default on 3.x.
-create /*+ noclustered */table t partitioned by (year, month) as
+create /*+ noclustered */table t partitioned by (year, month) stored as textfile as
select * from functional.alltypes
---- DISTRIBUTEDPLAN
Max Per-Host Resource Reservation: Memory=32.00KB Threads=2
@@ -617,7 +617,7 @@ max-parallelism=10 segment-costs=[62944]
# in insert.test.
# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
# because IMPALA-5293 changed clustered to be the default on 3.x.
-create /*+ noshuffle, noclustered */ table t partitioned by (year, month) as
+create /*+ noshuffle, noclustered */ table t partitioned by (year, month) stored as textfile as
select * from functional.alltypes
---- DISTRIBUTEDPLAN
Max Per-Host Resource Reservation: Memory=32.00KB Threads=2
@@ -650,7 +650,7 @@ max-parallelism=10 segment-costs=[62944]
in pipelines: 00(GETNEXT)
====
# CTAS with more complex select query
-create table t partitioned by (c_nationkey) sort by (c_custkey) as
+create table t partitioned by (c_nationkey) sort by (c_custkey) stored as textfile as
select c_custkey, max(o_totalprice) as maxprice, c_nationkey
from tpch.orders join tpch.customer on c_custkey = o_custkey
where c_nationkey < 10
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/ddl.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/ddl.test
index cf91fa35d..449f09467 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/ddl.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/ddl.test
@@ -1,5 +1,5 @@
# Some of tests here are copied from ../ddl.test
-create table t as select * from functional.alltypes
+create table t stored as textfile as select * from functional.alltypes
---- PLAN
Max Per-Host Resource Reservation: Memory=4.03MB Threads=1
Per-Host Resource Estimates: Memory=20MB
@@ -79,7 +79,7 @@ max-parallelism=10 segment-costs=[62944]
tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
in pipelines: 00(GETNEXT)
====
-create table t as
+create table t stored as textfile as
select distinct a.int_col, a.string_col from functional.alltypes a
inner join functional.alltypessmall b on (a.id = b.id)
where a.year = 2009 and b.month = 2
@@ -319,7 +319,7 @@ max-parallelism=10 segment-costs=[9666, 20] cpu-comparison-result=11 [max(10 (se
in pipelines: 00(GETNEXT)
====
# CTAS with a view that has a limit clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select int_col from functional.hive_view
---- PLAN
Max Per-Host Resource Reservation: Memory=32.00KB Threads=1
@@ -376,7 +376,7 @@ max-parallelism=10 segment-costs=[17343]
in pipelines: 00(GETNEXT)
====
# CTAS with multiple nested inline views that have a limit clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from
(select * from
(select int_col from functional.alltypestiny limit 1) v1 ) v2
@@ -436,7 +436,7 @@ max-parallelism=4 segment-costs=[16]
====
# CTAS with a select statement that has a limit and offset clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from functional.alltypestiny order by id limit 1 offset 5
---- PLAN
Max Per-Host Resource Reservation: Memory=4.01MB Threads=1
@@ -533,7 +533,7 @@ max-parallelism=4 segment-costs=[29, 0]
in pipelines: 00(GETNEXT)
====
# CTAS with an inline view that has a limit and offset clause (IMPALA-1411)
-create table t as
+create table t stored as textfile as
select * from
(select int_col from functional.alltypestiny order by id limit 2 offset 5) v
---- PLAN
@@ -631,7 +631,7 @@ max-parallelism=4 segment-costs=[29, 0]
in pipelines: 00(GETNEXT)
====
# CTAS with sort columns
-create table t sort by (int_col, bool_col) as
+create table t sort by (int_col, bool_col) stored as textfile as
select * from functional.alltypes
---- PLAN
Max Per-Host Resource Reservation: Memory=16.03MB Threads=1
@@ -725,7 +725,7 @@ max-parallelism=10 segment-costs=[62944]
in pipelines: 00(GETNEXT)
====
# CTAS with partitions and sort columns
-create table t partitioned by (year, month) sort by (int_col, bool_col) as
+create table t partitioned by (year, month) sort by (int_col, bool_col) stored as textfile as
select * from functional.alltypes
---- PLAN
Max Per-Host Resource Reservation: Memory=16.03MB Threads=1
@@ -824,7 +824,7 @@ max-parallelism=10 segment-costs=[62944]
# noshuffle hint would lead to a different plan.
# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
# because IMPALA-5293 changed clustered to be the default on 3.x.
-create /*+ noclustered */table t partitioned by (year, month) as
+create /*+ noclustered */table t partitioned by (year, month) stored as textfile as
select * from functional.alltypes
---- DISTRIBUTEDPLAN
Max Per-Host Resource Reservation: Memory=4.03MB Threads=2
@@ -875,7 +875,7 @@ max-parallelism=10 segment-costs=[62944]
# in insert.test.
# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
# because IMPALA-5293 changed clustered to be the default on 3.x.
-create /*+ noshuffle, noclustered */ table t partitioned by (year, month) as
+create /*+ noshuffle, noclustered */ table t partitioned by (year, month) stored as textfile as
select * from functional.alltypes
---- DISTRIBUTEDPLAN
Max Per-Host Resource Reservation: Memory=4.03MB Threads=2
@@ -922,7 +922,7 @@ max-parallelism=10 segment-costs=[62944]
in pipelines: 00(GETNEXT)
====
# CTAS with more complex select query
-create table t partitioned by (c_nationkey) sort by (c_custkey) as
+create table t partitioned by (c_nationkey) sort by (c_custkey) stored as textfile as
select c_custkey, max(o_totalprice) as maxprice, c_nationkey
from tpch.orders join tpch.customer on c_custkey = o_custkey
where c_nationkey < 10
diff --git a/testdata/workloads/functional-query/queries/QueryTest/acid-compute-stats.test b/testdata/workloads/functional-query/queries/QueryTest/acid-compute-stats.test
index 07c361fde..698c40a4f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/acid-compute-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/acid-compute-stats.test
@@ -2,7 +2,7 @@
---- QUERY
# Test unpartitioned table.
set DEFAULT_TRANSACTIONAL_TYPE=insert_only;
-create table tt (i int);
+create table tt (i int) stored as textfile;
insert into tt values (1);
compute stats tt;
====
@@ -27,7 +27,7 @@ STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
---- QUERY
# Test partitioned table with non-incremental stats.
set DEFAULT_TRANSACTIONAL_TYPE=insert_only;
-create table pt (x int) partitioned by (p int);
+create table pt (x int) partitioned by (p int) stored as textfile;
insert into pt partition (p=1) values (1);
compute stats pt;
====
@@ -67,7 +67,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
# recreated instead.
set DEFAULT_TRANSACTIONAL_TYPE=insert_only;
drop table pt;
-create table pt (x int) partitioned by (p int);
+create table pt (x int) partitioned by (p int) stored as textfile;
insert into pt partition (p=1) values (1);
compute incremental stats pt;
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test
index b652ffad0..bc4c83b17 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table-hdfs-caching.test
@@ -1,7 +1,7 @@
# This test file covers alter table statements that add multiple partitions and use hdfs caching
====
---- QUERY
-create table i1670B_alter (s string) partitioned by (i integer);
+create table i1670B_alter (s string) partitioned by (i integer) stored as textfile;
alter table i1670B_alter add
partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670B_alter/i1'
cached in 'testPool' with replication=3
@@ -18,7 +18,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# IMPALA-1670: Set up i1670C_alter table for the next test case.
-create table i1670C_alter (s string) partitioned by (i integer);
+create table i1670C_alter (s string) partitioned by (i integer) stored as textfile;
alter table i1670C_alter add
partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670C_alter/i2A'
cached in 'testPool' with replication=2
@@ -55,7 +55,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
# IMPALA-1670: Partitions without explicit CACHED IN/UNCACHED clause inherit cacheop from
# the parent table
create table i1670D_alter (s string) partitioned by (i integer)
-cached in 'testPool' with replication=7;
+stored as textfile cached in 'testPool' with replication=7;
alter table i1670D_alter add
partition (i=1) cached in 'testPool' with replication=5
partition (i=2)
@@ -74,7 +74,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
---- QUERY
# IMPALA-1670: After INVALIDATE METADATA Impala can access previously added partitions and
# partition data.
-create table i1670E_alter (a int) partitioned by (x int);
+create table i1670E_alter (a int) partitioned by (x int) stored as textfile;
alter table i1670E_alter add partition (x=1)
partition (x=2) uncached
partition (x=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670E_alter/x3'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
index e02034111..d1409024d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
@@ -2,6 +2,7 @@
---- QUERY
# First create an unpartitioned table
create external table t1 (i int)
+stored as textfile
location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/t1_tmp1'
---- RESULTS
'Table has been created.'
@@ -355,6 +356,7 @@ bigint,string,int,int
# Create a partitioned table. Specify the location so we know what dirs need
# to be cleaned after the test finishes executing.
create external table t_part (i int) partitioned by (j int, s string)
+stored as textfile
location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/t_part_tmp';
alter table t_part add partition (j=cast(2-1 as int), s='2012');
alter table t_part add if not exists partition (j=1, s='2012');
@@ -861,7 +863,7 @@ bigint,bigint
# Tests that renaming a table with column stats across databases
# preserves table and column stats, and allows the renamed table
# to be dropped (IMPALA-1711/HIVE-9720).
-create table $DATABASE.mv (x int, y string);
+create table $DATABASE.mv (x int, y string) stored as textfile;
insert into $DATABASE.mv values(1, 'a'), (2, 'b'), (NULL, NULL);
compute stats $DATABASE.mv;
alter table $DATABASE.mv rename to $DATABASE2.mv2;
@@ -915,7 +917,7 @@ STRING
# to be dropped (IMPALA-2810).
#
# Skip in Hive 3, see HIVE-21806 for details.
-create table $DATABASE.mv (x int) partitioned by (y string);
+create table $DATABASE.mv (x int) partitioned by (y string) stored as textfile;
insert into $DATABASE.mv partition(y='a') values(1);
insert into $DATABASE.mv partition(y='b') values(2);
insert into $DATABASE.mv partition(y=NULL) values(NULL);
@@ -1187,7 +1189,7 @@ INT, STRING
====
---- QUERY
# IMPALA-1670: Support adding multiple partitions in ALTER TABLE ADD PARTITION
-create table i1670A_alter (s string) partitioned by (i integer);
+create table i1670A_alter (s string) partitioned by (i integer) stored as textfile;
alter table i1670A_alter add
partition (i=1) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter/i1'
partition (i=2) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter/i2'
@@ -1204,7 +1206,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
---- QUERY
# IMPALA-1670: After INVALIDATE METADATA Impala can access previously added partitions and
# partition data.
-create table i1670A_alter_1 (a int) partitioned by (x int);
+create table i1670A_alter_1 (a int) partitioned by (x int) stored as textfile;
alter table i1670A_alter_1 add partition (x=1)
partition (x=2) uncached
partition (x=3) location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/i1670A_alter_1/x3';
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats-incremental.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-incremental.test
index 4e8400e6d..0a80bb5d0 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/compute-stats-incremental.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-incremental.test
@@ -158,7 +158,7 @@ COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE, #TRUES, #FALSES
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
====
---- QUERY
-create table incremental_empty_partitioned (i int) partitioned by (j int);
+create table incremental_empty_partitioned (i int) partitioned by (j int) stored as textfile;
alter table incremental_empty_partitioned add partition (j=1);
====
---- QUERY
@@ -365,7 +365,8 @@ STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING,
---- QUERY
# Test that many partition keys work correctly
create table incremental_many_part_keys(col int)
-partitioned by (p1 int, p2 int, p3 int, p4 int, p5 int, p6 int);
+partitioned by (p1 int, p2 int, p3 int, p4 int, p5 int, p6 int)
+stored as textfile;
insert into incremental_many_part_keys
partition(p1=1, p2=2, p3=3, p4=4, p5=5, p6=6) values(1);
====
@@ -379,7 +380,7 @@ show table stats incremental_many_part_keys;
STRING, STRING, STRING, STRING, STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
-create table incremental_null_part_key(col int) partitioned by (p int);
+create table incremental_null_part_key(col int) partitioned by (p int) stored as textfile;
insert into incremental_null_part_key partition(p) values(1,NULL), (1,2);
compute incremental stats incremental_null_part_key partition(p=2);
====
@@ -396,7 +397,8 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
---- QUERY
# Check that incremental stats queries handle partitions with keyword names
create table incremental_keyword_part_key(col int)
-partitioned by (`date` int, `table` string);
+partitioned by (`date` int, `table` string)
+stored as textfile;
insert into incremental_keyword_part_key
partition(`date`=1, `table`='a') values(2);
compute incremental stats incremental_keyword_part_key
@@ -439,7 +441,7 @@ STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING,
====
---- QUERY
create table incremental_string_part_value(col int) partitioned by
-(p string);
+(p string) stored as textfile;
insert into incremental_string_part_value partition(p="test_string")
values(2);
compute incremental stats incremental_string_part_value
@@ -454,7 +456,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
---- QUERY
# Check that dropping a column still allows stats to be computed
create table incremental_drop_column (a int, b int, c int)
-partitioned by (d int);
+partitioned by (d int) stored as textfile;
alter table incremental_drop_column add partition (d=1);
insert into incremental_drop_column partition(d=1) values (4,4,4);
compute incremental stats incremental_drop_column;
@@ -470,7 +472,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Check that adding a column invalidates all incremental stats
-create table incremental_add_column (col int) partitioned by (p int);
+create table incremental_add_column (col int) partitioned by (p int) stored as textfile;
insert into incremental_add_column partition(p) values(1,1),(2,2);
compute incremental stats incremental_add_column;
insert into incremental_add_column partition(p) values(1,1);
@@ -485,7 +487,7 @@ show table stats incremental_add_column;
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
-create table incremental_no_partitions (col int) partitioned by (p int);
+create table incremental_no_partitions (col int) partitioned by (p int) stored as textfile;
compute incremental stats incremental_no_partitions;
show table stats incremental_no_partitions;
---- RESULTS
@@ -494,7 +496,7 @@ show table stats incremental_no_partitions;
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
-create table incremental_not_partitioned (col int);
+create table incremental_not_partitioned (col int) stored as textfile;
insert into incremental_not_partitioned values(1),(2);
compute incremental stats incremental_not_partitioned;
show table stats incremental_not_partitioned;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
index 2f5ed1f5d..80c289511 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
@@ -666,7 +666,7 @@ STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
====
---- QUERY
# IMPALA-883: Compute table stats for an empty partition.
-create table empty_partitioned (i int) partitioned by (j int);
+create table empty_partitioned (i int) partitioned by (j int) stored as textfile;
alter table empty_partitioned add partition (j=1);
====
---- QUERY
@@ -730,7 +730,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# IMPALA-1614 Verify that COMPUTE STATS works on a table whose name starts with numbers.
-create table `123_table` (i int, 1p int) partitioned by (2j int);
+create table `123_table` (i int, 1p int) partitioned by (2j int) stored as textfile;
alter table `123_table` add partition (2j=1);
====
---- QUERY
diff --git a/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test b/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test
index 584a25eef..25596dd9a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test
@@ -1,6 +1,6 @@
====
---- QUERY
-create table corrupted (id int, name string) partitioned by (org int);
+create table corrupted (id int, name string) partitioned by (org int) stored as textfile;
====
---- QUERY
insert into corrupted partition (org=1) values (1, "Martin"), (2, "Hans"), (3, "Peter");
@@ -161,7 +161,7 @@ explain select count(*) from corrupted where org = 2;
STRING
====
---- QUERY
-create table corrupted_no_part (id int);
+create table corrupted_no_part (id int) stored as textfile;
insert into corrupted_no_part values (1),(2),(3);
compute stats corrupted_no_part;
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/create-table-like-table.test b/testdata/workloads/functional-query/queries/QueryTest/create-table-like-table.test
index 223ca9eea..cd31f9804 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/create-table-like-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/create-table-like-table.test
@@ -63,7 +63,7 @@ show table stats like_view
---- LABELS
#ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION, EC POLICY
---- RESULTS
--1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*,'$ERASURECODE_POLICY'
+-1,0,'0B','NOT CACHED','NOT CACHED','PARQUET','false',regex:.*,'$ERASURECODE_POLICY'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/grant_revoke.test b/testdata/workloads/functional-query/queries/QueryTest/grant_revoke.test
index 75217411a..af2e09e2c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/grant_revoke.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/grant_revoke.test
@@ -291,7 +291,8 @@ STRING
====
---- QUERY
# IMPALA-1670: User does not have privileges to access URI when adding partitions
-create table grant_rev_db.test_tbl_partitioned(i int) partitioned by (j int);
+create table grant_rev_db.test_tbl_partitioned(i int) partitioned by (j int)
+stored as textfile;
alter table grant_rev_db.test_tbl_partitioned add
partition (j=1)
partition (j=2) location '$FILESYSTEM_PREFIX/test-warehouse/grant_rev_test_prt';
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test b/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
index e6c2261fe..931e87191 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
@@ -3,7 +3,7 @@
use cachedb
====
---- QUERY
-create table cached_tbl_nopart (i int) cached in 'testPool'
+create table cached_tbl_nopart (i int) stored as textfile cached in 'testPool'
====
---- QUERY
insert into cached_tbl_nopart select 1
@@ -52,7 +52,7 @@ BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
drop table if exists cached_tbl_part
====
---- QUERY
-create table cached_tbl_part (i int) partitioned by (j int) cached in 'testPool' with replication = 9
+create table cached_tbl_part (i int) partitioned by (j int) stored as textfile cached in 'testPool' with replication = 9
====
---- QUERY
# new partition should inherit the cached property
@@ -79,7 +79,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
drop table if exists cached_tbl_part
====
---- QUERY
-create table cached_tbl_part (i int) partitioned by (j int) cached in 'testPool'
+create table cached_tbl_part (i int) partitioned by (j int) stored as textfile cached in 'testPool'
====
---- QUERY
# new partition should inherit the cached property
@@ -339,7 +339,7 @@ STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING, STRING
---- QUERY
# Create a table which resides on the local file system
create table uncached_tbl_local(i int)
-location 'file://$IMPALA_HOME/testdata/data/uncached_tbl_local'
+stored as textfile location 'file://$IMPALA_HOME/testdata/data/uncached_tbl_local'
====
---- QUERY
# Turning on caching should fail for this table
@@ -349,7 +349,7 @@ Table cachedb.uncached_tbl_local cannot be cached. Please check if the table or
====
---- QUERY
# Create a new table on HDFS
-create table cached_tbl_local(i int) partitioned by (j int)
+create table cached_tbl_local(i int) partitioned by (j int) stored as textfile
====
---- QUERY
# Add a partition which resides on the local filesystem
diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert_null.test b/testdata/workloads/functional-query/queries/QueryTest/insert_null.test
index 1f18b4e8b..abb267256 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/insert_null.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/insert_null.test
@@ -8,11 +8,13 @@ CREATE TABLE nullinsert (
int_cal int
)
row format delimited fields terminated by ',' escaped by '\\'
+stored as textfile
LOCATION '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/nullinsert';
CREATE EXTERNAL TABLE nullinsert_alt(
whole_row string
)
row format delimited fields terminated by '|'
+stored as textfile
LOCATION '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/nullinsert';
create table alltypesinsert like $ORIGINAL_DB.alltypesinsert;
create table nullformat_custom like $ORIGINAL_DB.nullformat_custom;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/local-filesystem.test b/testdata/workloads/functional-query/queries/QueryTest/local-filesystem.test
index 1239eea3e..8c9a8128e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/local-filesystem.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/local-filesystem.test
@@ -3,6 +3,7 @@
create external table tbl
(c0 string, c1 string, c2 string, c3 int, c4 int, c5 int)
row format delimited fields terminated by ','
+stored as textfile
location 'file://$IMPALA_HOME/testdata/data/local_tbl'
---- RESULTS
'Table has been created.'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test b/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test
index 8b1ff8803..49a7cc3d2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test
@@ -12,7 +12,9 @@ PARTITIONED BY (
float_col FLOAT,
double_col DOUBLE,
string_col STRING
-) LOCATION '$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE.db/all_insert_partition_col_types'
+)
+STORED AS TEXTFILE
+LOCATION '$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE.db/all_insert_partition_col_types'
====
---- QUERY
DESCRIBE all_insert_partition_col_types
@@ -224,7 +226,7 @@ DROP TABLE all_insert_partition_col_types;
# Regression test for IMPALA-1026
drop table if exists test_dec_partition;
create table test_dec_partition(id int, `dec` decimal(5,4))
- partitioned by(decimal_col DECIMAL(5,4));
+ partitioned by(decimal_col DECIMAL(5,4)) stored as textfile;
alter table test_dec_partition drop if exists partition(decimal_col=4.34);
insert into test_dec_partition partition(decimal_col=4.34) values (1, 3.14);
select * from test_dec_partition;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
index a70948921..31384e888 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
@@ -143,9 +143,9 @@ alter table p1 add partition (j=2,k="F");
show partitions p1
---- RESULTS
'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g,'$ERASURECODE_POLICY'
-'2','D',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=D,'$ERASURECODE_POLICY'
-'2','E',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=E,'$ERASURECODE_POLICY'
-'2','F',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=F,'$ERASURECODE_POLICY'
+'2','D',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=D,'$ERASURECODE_POLICY'
+'2','E',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=E,'$ERASURECODE_POLICY'
+'2','F',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=F,'$ERASURECODE_POLICY'
'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d,'$ERASURECODE_POLICY'
'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e,'$ERASURECODE_POLICY'
'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f,'$ERASURECODE_POLICY'
@@ -163,9 +163,9 @@ alter table p1 partition (j=100) set location '$FILESYSTEM_PREFIX/test-warehouse
show partitions p1
---- RESULTS
'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g,'$ERASURECODE_POLICY'
-'2','D',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=D,'$ERASURECODE_POLICY'
-'2','E',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=E,'$ERASURECODE_POLICY'
-'2','F',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=F,'$ERASURECODE_POLICY'
+'2','D',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=D,'$ERASURECODE_POLICY'
+'2','E',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=E,'$ERASURECODE_POLICY'
+'2','F',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=F,'$ERASURECODE_POLICY'
'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d,'$ERASURECODE_POLICY'
'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e,'$ERASURECODE_POLICY'
'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f,'$ERASURECODE_POLICY'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test
index 1b11e078a..1b5dd0c6c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/set.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/set.test
@@ -25,7 +25,7 @@ set all;
'PARQUET_FILE_SIZE','0','ADVANCED'
'REQUEST_POOL','','REGULAR'
'SYNC_DDL','0','REGULAR'
-'DEFAULT_FILE_FORMAT','TEXT','REGULAR'
+'DEFAULT_FILE_FORMAT','PARQUET','REGULAR'
'DISABLE_HDFS_NUM_ROWS_ESTIMATE','0','REGULAR'
---- TYPES
STRING, STRING, STRING
@@ -52,7 +52,7 @@ set all;
'PARQUET_FILE_SIZE','0','ADVANCED'
'REQUEST_POOL','','REGULAR'
'SYNC_DDL','0','REGULAR'
-'DEFAULT_FILE_FORMAT','TEXT','REGULAR'
+'DEFAULT_FILE_FORMAT','PARQUET','REGULAR'
'DISABLE_HDFS_NUM_ROWS_ESTIMATE','0','REGULAR'
---- TYPES
STRING, STRING, STRING
@@ -79,7 +79,7 @@ set all;
'PARQUET_FILE_SIZE','0','ADVANCED'
'REQUEST_POOL','','REGULAR'
'SYNC_DDL','0','REGULAR'
-'DEFAULT_FILE_FORMAT','TEXT','REGULAR'
+'DEFAULT_FILE_FORMAT','PARQUET','REGULAR'
'DISABLE_HDFS_NUM_ROWS_ESTIMATE','0','REGULAR'
---- TYPES
STRING, STRING, STRING
diff --git a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
index f5e48af33..0855dc70d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
@@ -417,7 +417,7 @@ CREATE TABLE show_create_table_test_db.i_1687_p (
PARTITIONED BY (
int_col INT
)
-STORED AS TEXTFILE
+STORED AS PARQUET
LOCATION '$$location_uri$$'
---- RESULTS-HIVE-3
CREATE EXTERNAL TABLE show_create_table_test_db.i_1687_p (
@@ -426,7 +426,7 @@ CREATE EXTERNAL TABLE show_create_table_test_db.i_1687_p (
PARTITIONED BY (
int_col INT
)
-STORED AS TEXTFILE
+STORED AS PARQUET
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'external.table.purge'='TRUE')
====
@@ -1150,7 +1150,7 @@ CREATE TABLE bucketed_test (a int, b string) CLUSTERED BY (a) into 4 buckets;
---- RESULTS-HIVE-3
CREATE EXTERNAL TABLE show_create_table_test_db.bucketed_test (a INT, b STRING)
CLUSTERED BY (a) INTO 4 BUCKETS
-STORED AS TEXTFILE
+STORED AS PARQUET
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'bucketing_version'='2',
'external.table.purge'='TRUE')
@@ -1161,7 +1161,7 @@ CREATE TABLE bucketed_sort_test (a int, b string) CLUSTERED BY (a) sort by (b) i
---- RESULTS-HIVE-3
CREATE EXTERNAL TABLE show_create_table_test_db.bucketed_sort_test (a INT, b STRING)
CLUSTERED BY (a) SORT BY LEXICAL (b) INTO 4 BUCKETS
-STORED AS TEXTFILE
+STORED AS PARQUET
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'bucketing_version'='2',
'external.table.purge'='TRUE')
diff --git a/testdata/workloads/functional-query/queries/QueryTest/union-const-scalar-expr-codegen.test b/testdata/workloads/functional-query/queries/QueryTest/union-const-scalar-expr-codegen.test
index 8db968980..5a43727c6 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/union-const-scalar-expr-codegen.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/union-const-scalar-expr-codegen.test
@@ -22,7 +22,7 @@ ExecOption: Codegen Enabled, Codegen Disabled for const scalar expressions
# Expect codegen to be disabled for const expressions.
set DISABLE_CODEGEN_ROWS_THRESHOLD=1;
drop table if exists test_values_codegen;
-create table test_values_codegen (c1 int, c2 timestamp, c3 string);
+create table test_values_codegen (c1 int, c2 timestamp, c3 string) stored as textfile;
insert into test_values_codegen(c1) values (CAST(1+ceil(2.5)*3 as tinyint));
---- RUNTIME_PROFILE
00:UNION
diff --git a/testdata/workloads/functional-query/queries/QueryTest/unsupported-writers.test b/testdata/workloads/functional-query/queries/QueryTest/unsupported-writers.test
index 68f355f84..c06c08003 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/unsupported-writers.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/unsupported-writers.test
@@ -1,6 +1,6 @@
====
---- QUERY
-create table __text_write (i int, s string, d double);
+create table __text_write (i int, s string, d double) stored as textfile;
====
---- QUERY
SET COMPRESSION_CODEC=NONE;
@@ -48,7 +48,7 @@ Writing to table format SEQUENCE_FILE is not supported.
# Test writing to mixed format table containing partitions in both supported and
# unsupported formats where writing to the partition with supported format should succeed.
# Create a table containing both text(supported) and avro(unsupported) partitions.
-create table __mixed_format_write (id int) partitioned by (part int);
+create table __mixed_format_write (id int) partitioned by (part int) stored as textfile;
====
---- QUERY
insert into __mixed_format_write partition(part=2000) values(1);
diff --git a/tests/comparison/db_connection.py b/tests/comparison/db_connection.py
index 601606253..bff73f68f 100644
--- a/tests/comparison/db_connection.py
+++ b/tests/comparison/db_connection.py
@@ -813,7 +813,7 @@ class ImpalaCursor(DbCursor):
'but does not have any primary keys'.format(
storage_format=table.storage_format))
- if table.storage_format != 'TEXTFILE':
+ if table.storage_format != 'PARQUET':
sql += "\nSTORED AS " + table.storage_format
if table.storage_location:
sql = sql.replace("CREATE TABLE", "CREATE EXTERNAL TABLE")
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index f6aa29e6c..8a9fac378 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -1387,9 +1387,13 @@ class TestExecutorGroups(CustomClusterTestSuite):
# The default query options for this test.
# Some test case will change these options along the test, but should eventually
# restored to this default values.
+ # This test's CpuAsk / fs-writer expectations are calibrated for CTAS statements
+ # producing TEXT tables, so pin DEFAULT_FILE_FORMAT=TEXT (IMPALA-2210 changed the
+ # global default to PARQUET).
self._set_query_options({
'COMPUTE_PROCESSING_COST': 'true',
- 'SLOT_COUNT_STRATEGY': 'PLANNER_CPU_ASK'})
+ 'SLOT_COUNT_STRATEGY': 'PLANNER_CPU_ASK',
+ 'DEFAULT_FILE_FORMAT': 'TEXT'})
# BEGIN testing insert + MAX_FS_WRITER
# Test unpartitioned insert, small scan, no MAX_FS_WRITER.
diff --git a/tests/custom_cluster/test_metadata_no_events_processing.py b/tests/custom_cluster/test_metadata_no_events_processing.py
index 59c1750db..a3cccd7fa 100644
--- a/tests/custom_cluster/test_metadata_no_events_processing.py
+++ b/tests/custom_cluster/test_metadata_no_events_processing.py
@@ -35,7 +35,8 @@ class TestMetadataNoEventsProcessing(CustomClusterTestSuite):
"""
tbl = unique_database + "." + "test"
self.client.execute(
- "create table {0} (c1 int) partitioned by (year int, month int)".format(tbl))
+ "create table {0} (c1 int) partitioned by (year int, month int)"
+ " stored as textfile".format(tbl))
# create 3 partitions and load data in them.
self.client.execute("insert into table {0} partition (year, month)"
"values (100, 2009, 1), (200, 2009, 2), (300, 2009, 3)".format(tbl))
@@ -81,7 +82,7 @@ class TestMetadataNoEventsProcessing(CustomClusterTestSuite):
assert len(result) == 4
# case2: change the partition to a different file-format, note that the table's
- # file-format is text.
+ # file-format is textfile (explicitly set).
# add another test partition. It should use the default file-format from the table.
self.execute_query("alter table {0} add partition (year=2020, month=9)".format(tbl))
# change the partition file-format from hive
diff --git a/tests/custom_cluster/test_tuple_cache.py b/tests/custom_cluster/test_tuple_cache.py
index 028bba0f5..a2d885038 100644
--- a/tests/custom_cluster/test_tuple_cache.py
+++ b/tests/custom_cluster/test_tuple_cache.py
@@ -129,7 +129,8 @@ class TestTupleCacheBase(CustomClusterTestSuite):
# Generates a table containing at least <scale> KB of data.
def create_table(self, fq_table, scale=1):
- self.execute_query("CREATE TABLE {0} ({1})".format(fq_table, TABLE_LAYOUT))
+ self.execute_query(
+ "CREATE TABLE {0} ({1}) STORED AS TEXTFILE".format(fq_table, TABLE_LAYOUT))
# To make the rows distinct, we keep using a different seed for table_value
global_index = 0
for _ in range(scale):
diff --git a/tests/data_errors/test_data_errors.py b/tests/data_errors/test_data_errors.py
index 005f6732b..5673ad13b 100644
--- a/tests/data_errors/test_data_errors.py
+++ b/tests/data_errors/test_data_errors.py
@@ -240,7 +240,7 @@ class TestTimestampErrors(TestDataErrors):
v.get_value('table_format').file_format == 'text')
def _setup_test_table(self, fq_tbl_name):
- create_stmt = "CREATE TABLE " + fq_tbl_name + " (col string)"
+ create_stmt = "CREATE TABLE " + fq_tbl_name + " (col string) STORED AS TEXTFILE"
insert_stmt = "INSERT INTO TABLE " + fq_tbl_name + " values" + \
"('1999-03-24 07:21:02'), ('2001-ån-02 12:12:15')," + \
"('1997-1131 02:09:32'), ('1954-12-03 15:10:02')," + \
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index e3b147359..5175781fb 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -376,7 +376,7 @@ class TestIncompatibleColStats(ImpalaTestSuite):
# Create a table with a string column and populate it with some data.
table_name = unique_database + ".badstats"
- self.client.execute("create table %s (s string)" % table_name)
+ self.client.execute("create table %s (s string) stored as textfile" % table_name)
self.client.execute("insert into table %s select cast(int_col as string) "
"from functional.alltypes limit 10" % table_name)
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 766f81775..338bb79d4 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -909,8 +909,9 @@ class TestDdlStatements(TestDdlBase):
def test_alter_tbl_properties_reload(self, unique_database):
# IMPALA-8734: Force a table schema reload when setting table properties.
tbl_name = "test_tbl"
- self.execute_query_expect_success(self.client, "create table {0}.{1} (c1 string)"
- .format(unique_database, tbl_name))
+ self.execute_query_expect_success(self.client,
+ "create table {0}.{1} (c1 string) stored as "
+ "textfile".format(unique_database, tbl_name))
self.filesystem_client.create_file("{2}/{0}.db/{1}/f".
format(unique_database, tbl_name, WAREHOUSE),
file_data="\nfoo\n")
@@ -936,13 +937,13 @@ class TestDdlStatements(TestDdlBase):
def test_create_table_file_format(self, unique_database):
# When default_file_format query option is not specified, the default table file
- # format is TEXT.
- text_table = "{0}.text_tbl".format(unique_database)
+ # format is PARQUET.
+ parquet_default_table = "{0}.parquet_default_tbl".format(unique_database)
self.execute_query_expect_success(
- self.client, "create table {0}(i int)".format(text_table))
+ self.client, "create table {0}(i int)".format(parquet_default_table))
result = self.execute_query_expect_success(
- self.client, "show create table {0}".format(text_table))
- assert any("TEXTFILE" in x for x in result.data)
+ self.client, "show create table {0}".format(parquet_default_table))
+ assert any("PARQUET" in x for x in result.data)
self.execute_query_expect_failure(
self.client, "create table {0}.foobar_tbl".format(unique_database),
diff --git a/tests/metadata/test_explain.py b/tests/metadata/test_explain.py
index 7a83ffc39..3426a4c09 100644
--- a/tests/metadata/test_explain.py
+++ b/tests/metadata/test_explain.py
@@ -116,7 +116,7 @@ class TestExplain(ImpalaTestSuite):
# Create a partitioned table with a mixed set of available stats,
mixed_tbl = unique_database + ".t"
self.execute_query(
- "create table %s (c int) partitioned by (p int)" % mixed_tbl)
+ "create table %s (c int) partitioned by (p int) stored as textfile" % mixed_tbl)
self.execute_query(
"insert into table %s partition (p) values(1,1),(2,2),(3,3)" % mixed_tbl)
# Set the number of rows at the table level.
@@ -165,7 +165,8 @@ class TestExplain(ImpalaTestSuite):
return self.execute_query("explain " + query, query_options={'explain_level': 3})
FQ_TBL_NAME = unique_database + ".t"
- self.execute_query("create table %s (i int, s string)" % FQ_TBL_NAME)
+ self.execute_query(
+ "create table %s (i int, s string) stored as textfile" % FQ_TBL_NAME)
# Fill the table with data that leads to avg_size of 4 for 's'.
self.execute_query("insert into %s values (1, '123'), (2, '12345')" % FQ_TBL_NAME)
@@ -197,7 +198,7 @@ class TestExplainEmptyPartition(ImpalaTestSuite):
corrupted, or used for something else."""
self.client.execute("SET EXPLAIN_LEVEL=3")
self.client.execute("CREATE TABLE %s.empty_partition (col int) "
- "partitioned by (p int)" % unique_database)
+ "partitioned by (p int) stored as textfile" % unique_database)
self.client.execute(
"ALTER TABLE %s.empty_partition ADD PARTITION (p=NULL)" % unique_database)
# Put an empty file in the partition so we have > 0 files, but 0 rows
diff --git a/tests/metadata/test_recover_partitions.py b/tests/metadata/test_recover_partitions.py
index 4815c7c89..f749c5e63 100644
--- a/tests/metadata/test_recover_partitions.py
+++ b/tests/metadata/test_recover_partitions.py
@@ -73,7 +73,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
NULL_INSERTED_VALUE = "4"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % (FQ_TBL_NAME))
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string) "
+ "STORED AS TEXTFILE" % (FQ_TBL_NAME))
self.execute_query_expect_success(self.client,
"INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % (FQ_TBL_NAME))
@@ -138,7 +139,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
INSERTED_VALUE = "4"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % FQ_TBL_NAME)
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string) "
+ "STORED AS TEXTFILE" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
"INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % FQ_TBL_NAME)
@@ -173,7 +175,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
DB_LOCATION = '%s/%s.db/' % (WAREHOUSE, unique_database)
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (s string)" % (FQ_TBL_NAME))
+ "CREATE TABLE %s (c int) PARTITIONED BY (s string) "
+ "STORED AS TEXTFILE" % (FQ_TBL_NAME))
# Create 700 partitions externally
try:
@@ -217,7 +220,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
FILE_PATH = "test"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % FQ_TBL_NAME)
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string) "
+ "STORED AS TEXTFILE" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
"INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % FQ_TBL_NAME)
@@ -268,7 +272,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
INSERTED_VALUE = "2"
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string)" % FQ_TBL_NAME)
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, p string) "
+ "STORED AS TEXTFILE" % FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
"INSERT INTO TABLE %s PARTITION(i=1, p='p1') VALUES(1)" % FQ_TBL_NAME)
@@ -311,6 +316,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
self.execute_query_expect_success(self.client,
"CREATE TABLE %s (i INT) PARTITIONED BY (a TINYINT, b SMALLINT, c INT, d BIGINT,"
" e DECIMAL(4,2), f FLOAT, g DOUBLE, j CHAR(5), k VARCHAR(6), s STRING)"
+ " STORED AS TEXTFILE"
% FQ_TBL_NAME)
self.execute_query_expect_success(self.client,
"INSERT INTO TABLE %s PARTITION(a=1, b=2, c=3, d=4, e=55.55, f=6.6, g=7.7, "
@@ -349,7 +355,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
FQ_TBL_NAME = unique_database + "." + TBL_NAME
self.execute_query_expect_success(
- self.client, "CREATE TABLE %s (s string) PARTITIONED BY (p string)" % FQ_TBL_NAME)
+ self.client, "CREATE TABLE %s (s string) PARTITIONED BY (p string)"
+ " STORED AS TEXTFILE" % FQ_TBL_NAME)
self.execute_query_expect_success(
self.client, "ALTER TABLE %s ADD PARTITION (p='100%%')" % FQ_TBL_NAME)
@@ -376,7 +383,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
tbl_location = self.__get_fs_location(unique_database, tbl_name)
self.execute_query_expect_success(
- self.client, "CREATE TABLE %s (i int) PARTITIONED BY (p string)" % fq_tbl_name)
+ self.client, "CREATE TABLE %s (i int) PARTITIONED BY (p string)"
+ " STORED AS TEXTFILE" % fq_tbl_name)
parts = ["\'", "\"", "\\\'", "\\\"", "\\\\\'", "\\\\\""]
for i in range(len(parts)):
# When creating partition directories, Hive replaces special characters in
@@ -400,7 +408,8 @@ class TestRecoverPartitions(ImpalaTestSuite):
TBL_LOCATION = self.__get_fs_location(unique_database, TBL_NAME)
self.execute_query_expect_success(self.client,
- "CREATE TABLE %s (c int) PARTITIONED BY (i int, s string)" % (FQ_TBL_NAME))
+ "CREATE TABLE %s (c int) PARTITIONED BY (i int, s string)"
+ " STORED AS TEXTFILE" % (FQ_TBL_NAME))
# Adds partition directories.
num_partitions = 10
diff --git a/tests/query_test/test_compressed_formats.py b/tests/query_test/test_compressed_formats.py
index 0ee022a1d..5679381d6 100644
--- a/tests/query_test/test_compressed_formats.py
+++ b/tests/query_test/test_compressed_formats.py
@@ -317,7 +317,7 @@ class TestReadZtsdLibCompressedFile(ImpalaTestSuite):
def __create_test_table(self, table_name, location):
self.client.execute("DROP TABLE IF EXISTS %s" % table_name)
- self.client.execute("CREATE TABLE %s (col string) LOCATION '%s'"
+ self.client.execute("CREATE TABLE %s (col string) STORED AS TEXTFILE LOCATION '%s'"
% (table_name, location))
def test_query_large_file(self):
diff --git a/tests/query_test/test_delimited_text.py b/tests/query_test/test_delimited_text.py
index 388b7bfe1..79383521b 100644
--- a/tests/query_test/test_delimited_text.py
+++ b/tests/query_test/test_delimited_text.py
@@ -86,7 +86,7 @@ class TestDelimitedText(ImpalaTestSuite):
"""IMPALA-13161: Verifies reading a large file which has full of field delimiters
won't causing crash due to overflows"""
tbl = unique_database + ".tbl"
- self.execute_query("create table {}(i int)".format(tbl))
+ self.execute_query("create table {}(i int) stored as textfile".format(tbl))
table_loc = self._get_table_location(tbl, vector)
# Generate a 3GB data file that has full of '\x00' (the default field delimiter)
with open("data.txt", "wb") as f:
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index 1b133b898..c8ad23b4b 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -231,7 +231,7 @@ class TestHdfsCachingDdl(ImpalaTestSuite):
num_entries_pre = get_num_cache_requests()
create_table = ("create table cachedb.cached_tbl_reload "
- "(id int) cached in 'testPool' with replication = 8")
+ "(id int) stored as textfile cached in 'testPool' with replication = 8")
self.execute_query_using_vector(create_table, vector)
# Access the table once to load the metadata
@@ -239,7 +239,8 @@ class TestHdfsCachingDdl(ImpalaTestSuite):
"select count(*) from cachedb.cached_tbl_reload", vector)
create_table = ("create table cachedb.cached_tbl_reload_part (i int) "
- "partitioned by (j int) cached in 'testPool' with replication = 8")
+ "partitioned by (j int) stored as textfile cached in 'testPool' "
+ "with replication = 8")
self.execute_query_using_vector(create_table, vector)
# Add two partitions
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 6cea1b5ef..70a918fa3 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -756,8 +756,8 @@ class TestHdfsParquetTableStatsWriter(ImpalaTestSuite):
table_name = "test_char_types"
qualified_table_name = "{0}.{1}".format(unique_database, table_name)
- create_table_stmt = "create table {0} (c3 char(3), vc varchar, st string);".format(
- qualified_table_name)
+ create_table_stmt = ("create table {0} (c3 char(3), vc varchar, st string)"
+ " stored as textfile;").format(qualified_table_name)
self.execute_query(create_table_stmt)
insert_stmt = """insert into {0} values
@@ -840,8 +840,8 @@ class TestHdfsParquetTableStatsWriter(ImpalaTestSuite):
table_name = "test_float_infinity"
qualified_table_name = "{0}.{1}".format(unique_database, table_name)
- create_table_stmt = "create table {0} (f float, d double);".format(
- qualified_table_name)
+ create_table_stmt = ("create table {0} (f float, d double)"
+ " stored as textfile;").format(qualified_table_name)
self.execute_query(create_table_stmt)
insert_stmt = """insert into {0} values
@@ -880,7 +880,8 @@ class TestHdfsParquetTableStatsWriter(ImpalaTestSuite):
table_name = "int96_nanos"
qualified_table_name = "{0}.{1}".format(unique_database, table_name)
- create_table_stmt = "create table {0} (ts timestamp);".format(qualified_table_name)
+ create_table_stmt = "create table {0} (ts timestamp) stored as textfile;".format(
+ qualified_table_name)
self.execute_query(create_table_stmt)
insert_stmt = """insert into {0} values
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 3761c9fc3..834a17da5 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -1504,7 +1504,8 @@ class TestTextSplitDelimiters(ImpalaTestSuite):
TABLE_NAME = "test_text_split_delimiters"
qualified_table_name = "%s.%s" % (unique_database, TABLE_NAME)
location = get_fs_path("/test-warehouse/%s.db/%s" % (unique_database, TABLE_NAME))
- query = "create table %s (s string) location '%s'" % (qualified_table_name, location)
+ query = "create table %s (s string) stored as textfile location '%s'" % (
+ qualified_table_name, location)
self.execute_query(query, options)
# Passing "w+" to NamedTemporaryFile prevents it from opening the file in bytes mode
@@ -1607,7 +1608,9 @@ class TestUncompressedText(ImpalaTestSuite):
# IMPALA-5315: Test support for date/time in unpadded format
def test_scan_lazy_timestamp(self, vector, unique_database):
test_files = ["testdata/data/lazy_timestamp.csv"]
- create_table_and_copy_files(self.client, """CREATE TABLE {db}.{tbl} (ts TIMESTAMP)""",
+ create_table_and_copy_files(self.client,
+ """CREATE TABLE {db}.{tbl} (ts TIMESTAMP)
+ STORED AS TEXTFILE""",
unique_database, "lazy_ts", test_files)
self.run_test_case('QueryTest/select-lazy-timestamp', vector, unique_database)
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index 4182983c6..1eba2a780 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -822,7 +822,7 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
assert "\tEXPLAIN_LEVEL: [STANDARD]" in result.stdout
assert "INVALID_QUERY_OPTION is not supported for the impalad being connected to, "\
"ignoring." in result.stdout
- assert "\tDEFAULT_FILE_FORMAT: [TEXT]" in result.stdout
+ assert "\tDEFAULT_FILE_FORMAT: [PARQUET]" in result.stdout
def test_commandline_flag_disable_live_progress(self, vector):
"""Test the command line flag disable_live_progress with live_progress."""
(impala) 03/03: IMPALA-15054: Set EXPAND_COMPLEX_TYPES to true by default
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c4fe178bac387c82dda0e79e48dced999d8a7fdf
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Tue Jun 30 15:03:25 2026 +0200
IMPALA-15054: Set EXPAND_COMPLEX_TYPES to true by default
Flips the default of the EXPAND_COMPLEX_TYPES query option from false to
true so that '*' expressions include complex-typed (ARRAY/MAP/STRUCT)
columns by default.
- common/thrift/Query.thrift: change the default to true.
- Docs: update the EXPAND_COMPLEX_TYPES default and rationale, rework
the complex-types SELECT * example, add an Impala 5.0.x
incompatible-changes entry, and clarify the Iceberg metadata-tables
note.
- SelectStmt.java: refresh stale comments (the empty-select-list
invariant in analyzeStarItem still holds under the new default).
Test handling for the behaviour change. Where a query specifically uses
the collection-join/unnest syntax, expands a struct via '.*', or is a
dedicated/paired complex-type test, EXPAND_COMPLEX_TYPES is pinned to
false so the query keeps its previous meaning; incidental top-level
'SELECT *' tests adopt the new default and their expected results are
regenerated.
Change-Id: I6cf01f0b6bb3b5fa2aa770c77bc772abf9cc84b2
Assisted-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
Reviewed-on: http://gerrit.cloudera.org:8080/24543
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
common/thrift/Query.thrift | 2 +-
docs/topics/impala_complex_types.xml | 34 +++++++------
docs/topics/impala_expand_complex_types.xml | 10 ++--
docs/topics/impala_iceberg.xml | 2 +-
docs/topics/impala_incompatible_changes.xml | 20 ++++++++
.../org/apache/impala/analysis/SelectStmt.java | 11 +++--
.../apache/impala/analysis/AnalyzeStmtsTest.java | 45 ++++++++++++-----
.../org/apache/impala/analysis/AnalyzerTest.java | 4 +-
.../org/apache/impala/planner/PlannerTest.java | 15 +++++-
.../queries/PlannerTest/empty.test | 2 +-
.../queries/QueryTest/iceberg-metadata-tables.test | 1 +
...migrated-table-field-id-resolution-complex.test | 8 ++++
.../queries/QueryTest/iceberg-query.test | 2 +-
.../queries/QueryTest/iceberg-v3-negative.test | 1 +
.../queries/QueryTest/max-nesting-depth.test | 12 +++++
.../QueryTest/nested-struct-in-select-list.test | 3 ++
.../QueryTest/nested-types-star-expansion.test | 4 ++
.../queries/QueryTest/nested-types-subplan.test | 4 ++
.../QueryTest/nested-types-tpch-mem-limit.test | 2 +
.../queries/QueryTest/paimon-query.test | 4 ++
.../ranger_column_masking_complex_types.test | 56 +++++++++++-----------
.../queries/QueryTest/ranger_row_filtering.test | 3 ++
.../virtual-column-file-position-parquet.test | 8 ++--
tests/custom_cluster/test_lineage.py | 4 ++
tests/query_test/test_kudu.py | 6 ++-
25 files changed, 186 insertions(+), 77 deletions(-)
diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift
index 8c76e36aa..dd8795e8a 100644
--- a/common/thrift/Query.thrift
+++ b/common/thrift/Query.thrift
@@ -656,7 +656,7 @@ struct TQueryOptions {
147: optional TSchemaResolutionStrategy orc_schema_resolution = 0;
// See comment in ImpalaService.thrift
- 148: optional bool expand_complex_types = false;
+ 148: optional bool expand_complex_types = true;
149: optional string fallback_db_for_functions;
diff --git a/docs/topics/impala_complex_types.xml b/docs/topics/impala_complex_types.xml
index f48b0d5f0..aeb50d9c8 100644
--- a/docs/topics/impala_complex_types.xml
+++ b/docs/topics/impala_complex_types.xml
@@ -1033,27 +1033,26 @@ STORED AS PARQUET;
</p>
<p>
- Because of backward compatibility with earlier versions of Impala that did not support
- complex types in the result set, queries using <codeph>SELECT *</codeph> skip complex
- types by default. To include complex types in <codeph>SELECT *</codeph> queries, set the
- EXPAND_COMPLEX_TYPES query option to true (see the
+ Queries using <codeph>SELECT *</codeph> include complex types in the result set by
+ default. To skip complex types in <codeph>SELECT *</codeph> queries (restoring the
+ behavior of earlier versions of Impala that did not support complex types in the result
+ set), set the EXPAND_COMPLEX_TYPES query option to false (see the
<xref href="impala_expand_complex_types.xml"/>).
</p>
<p>
- The following example shows how referring directly to a column with a complex type where
- a struct is embedded in a collection (an array) returns an error, while
- <codeph>SELECT *</codeph> on the same table succeeds, but only retrieves the scalar
- columns. Note that if EXPAND_COMPLEX_TYPES is true, the <codeph>SELECT *</codeph> query
- also fails with the same error.
+ The following example shows how the <codeph>EXPAND_COMPLEX_TYPES</codeph> query option
+ affects <codeph>SELECT *</codeph>. Complex-typed columns can appear in a query result,
+ but Impala cannot write them into another table. Because
+ <codeph>EXPAND_COMPLEX_TYPES</codeph> defaults to true, <codeph>SELECT *</codeph>
+ includes the complex <codeph>c_orders</codeph> column, so a <codeph>CREATE TABLE ... AS
+ SELECT *</codeph> fails. Setting the option to false makes <codeph>SELECT *</codeph> skip
+ complex columns, so the statement copies only the scalar columns.
</p>
<note conref="../shared/impala_common.xml#common/complex_type_schema_pointer"/>
-<codeblock><![CDATA[SELECT c_orders FROM customer LIMIT 1;
-ERROR: AnalysisException: STRUCT type inside collection types is not supported.
-
--- Original column has several scalar and one complex column.
+<codeblock><![CDATA[-- The customer table has several scalar columns and one complex column, c_orders.
DESCRIBE customer;
+--------------+------------------------------------+
| name | type |
@@ -1069,7 +1068,14 @@ DESCRIBE customer;
| | >> |
+--------------+------------------------------------+
--- When we SELECT * from that table, only the scalar columns come back in the result set.
+-- EXPAND_COMPLEX_TYPES defaults to true, so SELECT * includes the c_orders complex column.
+-- Impala cannot store a complex-typed column in a table, so CREATE TABLE AS SELECT * fails.
+CREATE TABLE select_star_customer STORED AS PARQUET AS SELECT * FROM customer;
+ERROR: AnalysisException: Unable to INSERT into target table (default.select_star_customer) because the column 'c_orders' has a complex type 'ARRAY<STRUCT<...>>' and Impala doesn't support inserting into tables containing complex type columns
+
+-- Setting EXPAND_COMPLEX_TYPES to false makes SELECT * skip complex columns, so only the
+-- scalar columns are copied.
+SET EXPAND_COMPLEX_TYPES=false;
CREATE TABLE select_star_customer STORED AS PARQUET AS SELECT * FROM customer;
+------------------------+
| summary |
diff --git a/docs/topics/impala_expand_complex_types.xml b/docs/topics/impala_expand_complex_types.xml
index f5de8257d..203c8c86f 100644
--- a/docs/topics/impala_expand_complex_types.xml
+++ b/docs/topics/impala_expand_complex_types.xml
@@ -38,10 +38,10 @@ under the License.
<indexterm audience="hidden">EXPAND_COMPLEX_TYPES Query Option</indexterm> Older versions of
Impala did not support complex types in the select list, and therefore a <codeph>SELECT
*</codeph> statement only expanded to primitive types even when the table contained complex-typed
- columns. Support for complex types in the select list has since been added, but for backward
- compatibility the behaviour that <q>*</q> expressions skip complex types has been kept as the
- default. If the EXPAND_COMPLEX_TYPES query option is set to true, <q>*</q> expressions will
- include complex types as well.
+ columns. Support for complex types in the select list has since been added, and <q>*</q>
+ expressions now include complex types by default. If the EXPAND_COMPLEX_TYPES query option is set
+ to false, <q>*</q> expressions skip complex types, restoring the behaviour of earlier Impala
+ versions.
</p>
<p>
@@ -50,7 +50,7 @@ under the License.
</p>
<p><b>Type:</b>BOOLEAN</p>
- <p><b>Default:</b>FALSE</p>
+ <p><b>Default:</b>TRUE</p>
<p><b>Added in:</b>Impala 4.2</p>
<p conref="../shared/impala_common.xml#common/related_info"/>
<p>
diff --git a/docs/topics/impala_iceberg.xml b/docs/topics/impala_iceberg.xml
index 914847ac9..ef6f656bb 100644
--- a/docs/topics/impala_iceberg.xml
+++ b/docs/topics/impala_iceberg.xml
@@ -1069,7 +1069,7 @@ DESCRIBE functional_parquet.iceberg_alltypes_part.history;
tables: for <codeph>SELECT *</codeph> expressions from metadata tables, complex
types will always be included, and for <codeph>SELECT *</codeph> expressions from
regular tables, complex types will be included if and only if
- <codeph>EXPAND_COMPLEX_TYPES</codeph> is true.
+ <codeph>EXPAND_COMPLEX_TYPES</codeph> is true (which is the default).
</p>
<p>
Note that unnesting collections from metadata tables is not supported.
diff --git a/docs/topics/impala_incompatible_changes.xml b/docs/topics/impala_incompatible_changes.xml
index 63991ac25..bd1832034 100644
--- a/docs/topics/impala_incompatible_changes.xml
+++ b/docs/topics/impala_incompatible_changes.xml
@@ -50,6 +50,26 @@ under the License.
</p>
<p outputclass="toc inpage"/>
</conbody>
+ <concept id="incompatible_changes_500x">
+ <title>Incompatible Changes Introduced in Impala 5.0.x</title>
+ <conbody>
+ <p>
+ <ul>
+ <li>The default value of the <codeph>EXPAND_COMPLEX_TYPES</codeph> query option
+ changed from <codeph>FALSE</codeph> to <codeph>TRUE</codeph>.<p>As a result,
+ <codeph>SELECT *</codeph> queries on tables that contain complex-typed
+ (<codeph>ARRAY</codeph>, <codeph>MAP</codeph>, or <codeph>STRUCT</codeph>)
+ columns now return those columns by default, instead of skipping them.</p><p>If
+ a table has a complex column whose type is not supported in the select list
+ (for example a <codeph>STRUCT</codeph> nested inside a collection), a
+ <codeph>SELECT *</codeph> that previously succeeded by skipping the column now
+ fails with an analysis error.</p><p>To restore the previous behavior, set the
+ <codeph>EXPAND_COMPLEX_TYPES</codeph> query option to <codeph>false</codeph>.
+ See <xref href="impala_expand_complex_types.xml"/>.</p></li>
+ </ul>
+ </p>
+ </conbody>
+ </concept>
<concept id="incompatible_changes_400x">
<title>Incompatible Changes Introduced in Impala 4.0.x</title>
<conbody>
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
index 1d2ed487c..9c766bae1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
@@ -867,9 +867,9 @@ public class SelectStmt extends QueryStmt {
/**
* Expand "*" select list item, ignoring semi-joined tables because those are
- * currently illegal in any select list (even for inline views, etc.). Also ignores
- * complex-typed fields for backwards compatibility unless EXPAND_COMPLEX_TYPES is set
- * to true.
+ * currently illegal in any select list (even for inline views, etc.). Complex-typed
+ * fields are included unless EXPAND_COMPLEX_TYPES is set to false (it defaults to
+ * true); see addStarExpandedPath().
*/
private void expandStar(SelectListItem selectListItem) throws AnalysisException {
if (fromClause_.isEmpty()) {
@@ -888,8 +888,9 @@ public class SelectStmt extends QueryStmt {
}
/**
- * Expand "path.*" from a resolved path, ignoring complex-typed fields for backwards
- * compatibility unless EXPAND_COMPLEX_TYPES is set to true.
+ * Expand "path.*" from a resolved path. Complex-typed fields are included unless
+ * EXPAND_COMPLEX_TYPES is set to false (it defaults to true);
+ * see addStarExpandedPath().
*/
private void expandStar(SelectListItem selectListItem, Path resolvedPath)
throws AnalysisException {
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index 5e0d1cca3..7b35502c7 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -949,8 +949,11 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalysisError("select t.a.a.a.* from a.a t",
"Cannot expand star in 't.a.a.a.*' because path 't.a.a.a' " +
"resolved to type 'INT'.");
- // Not ambiguous, but expands to an empty select list.
- AnalysisError("select t.* from a.a t",
+ // Not ambiguous, but expands to an empty select list when complex types are not
+ // expanded into '*'. EXPAND_COMPLEX_TYPES defaults to true, so set it to false here.
+ AnalysisContext ctxNoExpand = createAnalysisCtx();
+ ctxNoExpand.getQueryOptions().setExpand_complex_types(false);
+ AnalysisError("select t.* from a.a t", ctxNoExpand,
"The star exprs expanded to an empty select list because the referenced " +
"tables only have complex-typed columns.");
@@ -1102,27 +1105,38 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
}
/**
- * Test that complex types are supported in the select list.
+ * Test that '*' skips complex-typed columns when EXPAND_COMPLEX_TYPES is disabled.
*/
@Test
- public void TestComplexTypesInSelectList() {
+ public void TestDisableExpandComplexTypes() {
+ // The following cases exercise the behavior where '*' skips complex-typed columns,
+ // which requires EXPAND_COMPLEX_TYPES to be disabled (it defaults to true).
+ AnalysisContext ctxNoExpand = createAnalysisCtx();
+ ctxNoExpand.getQueryOptions().setExpand_complex_types(false);
// Star only expands to the scalar-typed fields.
AnalyzesOk("select * from functional.allcomplextypes " +
- "cross join functional_parquet.alltypes");
- AnalyzesOk("select complex_struct_col.* from functional.allcomplextypes");
+ "cross join functional_parquet.alltypes", ctxNoExpand);
+ AnalyzesOk("select complex_struct_col.* from functional.allcomplextypes",
+ ctxNoExpand);
// The result exprs could be empty after star expansion.
addTestTable("create table only_complex_types " +
"(a array<int>, b struct<x:int, y:int>, c map<string, int>)");
- AnalysisError("select * from only_complex_types",
+ AnalysisError("select * from only_complex_types", ctxNoExpand,
"The star exprs expanded to an empty select list because the referenced " +
"tables only have complex-typed columns.");
AnalysisError("select a.* from only_complex_types a, " +
- "functional.allcomplextypes b",
+ "functional.allcomplextypes b", ctxNoExpand,
"The star exprs expanded to an empty select list because the referenced " +
"tables only have complex-typed columns.");
// Empty star expansion, but non empty result exprs.
- AnalyzesOk("select 1, * from only_complex_types");
+ AnalyzesOk("select 1, * from only_complex_types", ctxNoExpand);
+ }
+ /**
+ * Test that complex types are supported in the select list.
+ */
+ @Test
+ public void TestComplexTypesInSelectList() {
// Struct in select list.
AnalysisContext ctx = createAnalysisCtx();
AnalyzesOk("select alltypes from functional_orc_def.complextypes_structs", ctx);
@@ -1175,9 +1189,8 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalyzesOk("select binary_key_col from functional_parquet.binary_in_complex_types");
AnalyzesOk("select binary_value_col from functional_parquet.binary_in_complex_types");
- //Make complex types available in star queries
- ctx.getQueryOptions().setExpand_complex_types(true);
-
+ // Complex types are available in star queries because EXPAND_COMPLEX_TYPES
+ // defaults to true.
AnalyzesOk("select * from functional_parquet.complextypes_structs",ctx);
AnalyzesOk("select * from functional_parquet.complextypes_nested_structs",ctx);
AnalyzesOk("select * from functional_parquet.complextypes_maps_view",ctx);
@@ -4784,8 +4797,14 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
"Unable to INSERT into target table (functional.allcomplextypes) because the " +
"column 'int_array_col' has a complex type 'ARRAY<INT>' and Impala doesn't " +
"support inserting into tables containing complex type columns");
+ // EXPAND_COMPLEX_TYPES defaults to true; disable it so that 'select *' does not
+ // expand the complex-typed columns of the (text-format) source table, which would
+ // otherwise fail with "Querying STRUCT is only supported for ORC and Parquet file
+ // formats" before the target-table complex-column check runs.
+ AnalysisContext ctxNoExpand = createAnalysisCtx();
+ ctxNoExpand.getQueryOptions().setExpand_complex_types(false);
AnalysisError("insert " + qualifier + " table functional.allcomplextypes " +
- "select * from functional.allcomplextypes",
+ "select * from functional.allcomplextypes", ctxNoExpand,
"Unable to INSERT into target table (functional.allcomplextypes) because the " +
"column 'int_array_col' has a complex type 'ARRAY<INT>' and Impala doesn't " +
"support inserting into tables containing complex type columns");
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
index d03cab7ca..1ebe85d89 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
@@ -346,8 +346,8 @@ public class AnalyzerTest extends FrontendTestBase {
AnalyzesOk(
"select id, input__file__name from functional_parquet.iceberg_partitioned");
AnalyzesOk(
- "select input__file__name, * from functional_parquet.complextypestbl c, " +
- "c.int_array");
+ "select input__file__name, * from functional_parquet.complextypestbl c, " +
+ "c.int_array arr");
AnalyzesOk(
"select c.input__file__name, c.int_array.* " +
"from functional_parquet.complextypestbl c, c.int_array");
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
index d13ea9da1..dda704a06 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
@@ -387,6 +387,9 @@ public class PlannerTest extends PlannerTestBase {
public void testNestedCollections() {
TQueryOptions options = new TQueryOptions();
options.setMinmax_filter_sorted_columns(false);
+ // This file's golden plans predate EXPAND_COMPLEX_TYPES defaulting to true; keep '*'
+ // skipping complex columns so the expected row sizes/output exprs stay valid.
+ options.setExpand_complex_types(false);
runPlannerTestFile("nested-collections", options);
}
@@ -576,7 +579,11 @@ public class PlannerTest extends PlannerTestBase {
@Test
public void testLineage() {
- runPlannerTestFile("lineage");
+ // This file's golden lineage predates EXPAND_COMPLEX_TYPES defaulting to true;
+ // keep '*' skipping complex columns so the expected lineage graph stays valid.
+ TQueryOptions options = defaultQueryOptions();
+ options.setExpand_complex_types(false);
+ runPlannerTestFile("lineage", options);
}
@Test
@@ -801,6 +808,9 @@ public class PlannerTest extends PlannerTestBase {
options.setMt_dop(3);
options.setDisable_hdfs_num_rows_estimate(true);
options.setExplain_level(TExplainLevel.EXTENDED);
+ // This file's golden plans predate EXPAND_COMPLEX_TYPES defaulting to true; keep '*'
+ // skipping complex columns so the expected output exprs stay valid.
+ options.setExpand_complex_types(false);
runPlannerTestFile("mt-dop-validation", options);
}
@@ -965,6 +975,9 @@ public class PlannerTest extends PlannerTestBase {
options.setMinmax_filter_threshold(0.0);
// Required so that output doesn't vary by whether parquet tables are used or not.
options.setDisable_hdfs_num_rows_estimate(true);
+ // These files' golden plans predate EXPAND_COMPLEX_TYPES defaulting to true; keep '*'
+ // skipping complex columns so the expected resources/output exprs stay valid.
+ options.setExpand_complex_types(false);
runPlannerTestFile(testFile, options,
ImmutableSet.of(PlannerTestOption.EXTENDED_EXPLAIN,
PlannerTestOption.INCLUDE_EXPLAIN_HEADER,
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/empty.test b/testdata/workloads/functional-planner/queries/PlannerTest/empty.test
index 9855f3d39..4d54804cc 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/empty.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/empty.test
@@ -606,7 +606,7 @@ PLAN-ROOT SINK
00:EMPTYSET
====
# IMPALA-5562: Relative table ref and limit 0.
-select * from tpch_nested_parquet.customer c, c.c_orders limit 0
+select * from tpch_nested_parquet.customer c, c.c_orders o limit 0
---- PLAN
PLAN-ROOT SINK
|
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
index bdf9fb041..373a0792c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test
@@ -1175,6 +1175,7 @@ INT,STRING,STRING,INT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,BI
---- QUERY
# Join a metadata table with a random other table that has complex columns and check that
# only the metadata table's complex columns are added to the result set.
+set expand_complex_types=false;
select * from functional_parquet.iceberg_query_metadata.`entries` ent
join functional_parquet.complextypes_arrays ca on ent.sequence_number = ca.id
where ca.id = 1;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrated-table-field-id-resolution-complex.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrated-table-field-id-resolution-complex.test
index edc578849..24ae26edc 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrated-table-field-id-resolution-complex.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrated-table-field-id-resolution-complex.test
@@ -35,6 +35,7 @@ select teststeps.step_description, teststeps.step_number from array_struct_table
STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from array_struct_table_test;
---- RESULTS
1,'Test 1','2024-08-26'
@@ -66,6 +67,7 @@ select teststeps.step_description, teststeps.step_number from array_struct_table
STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from array_struct_table_test;
---- RESULTS
1,'Test 1','2024-08-26'
@@ -111,6 +113,7 @@ select m.result_date, teststeps.step_description, teststeps.step_number from arr
STRING, STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from array_struct_table_test;
---- RESULTS
3,'Test 3','2024-08-27'
@@ -145,6 +148,7 @@ select teststeps.step_description, teststeps.step_number from array_struct_table
STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from array_struct_table_test_orc;
---- RESULTS
1,'Test 1','2024-08-26'
@@ -176,6 +180,7 @@ select teststeps.step_description, teststeps.step_number from array_struct_table
STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from array_struct_table_test_orc;
---- RESULTS
1,'Test 1','2024-08-26'
@@ -221,6 +226,7 @@ select m.result_date, teststeps.step_description, teststeps.step_number from arr
STRING, STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from array_struct_table_test_orc;
---- RESULTS
4,'Test 4','2024-08-27'
@@ -275,6 +281,7 @@ select teststeps.step_description, teststeps.step_number from all_part_cols_stor
STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from all_part_cols_stored_parquet;
---- RESULTS
1,'Test 1','2024-08-26'
@@ -308,6 +315,7 @@ select teststeps.step_description, teststeps.step_number from all_part_cols_stor
STRING, INT
====
---- QUERY
+set expand_complex_types=false;
select * from all_part_cols_stored_orc;
---- RESULTS
1,'Test 1','2024-08-26'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
index 49e6c100c..d83fc117c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
@@ -519,7 +519,7 @@ string, string, string
SELECT * FROM iceberg_uppercase_col;
---- RESULTS
---- TYPES
-STRING
+STRING,STRING
====
---- QUERY
SET TIMEZONE='UTC';
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
index 9856e2eba..9eaf9413a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
@@ -57,6 +57,7 @@ Default values are not supported for TIMESTAMPTZ types
====
---- QUERY
# SELECT * materializes ts_tz, so timestamptz default rejection applies.
+set expand_complex_types=false;
SELECT * FROM test_complex_default;
---- CATCH
Default values are not supported for TIMESTAMPTZ types
diff --git a/testdata/workloads/functional-query/queries/QueryTest/max-nesting-depth.test b/testdata/workloads/functional-query/queries/QueryTest/max-nesting-depth.test
index eed7eacb2..2c54e7fef 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/max-nesting-depth.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/max-nesting-depth.test
@@ -14,6 +14,9 @@ int
====
---- QUERY
# Test absolute table ref executed with a single scan.
+# EXPAND_COMPLEX_TYPES defaults to true; set it to false so that 'select *' returns the
+# deeply nested scalar item instead of expanding the complex-typed columns.
+set expand_complex_types=false;
select * from $DATABASE.int_array_tbl.f.
item.item.item.item.item.item.item.item.item.item.
item.item.item.item.item.item.item.item.item.item.
@@ -32,6 +35,9 @@ int
====
---- QUERY
# Test relative ref executed with deeply nested subplans.
+# EXPAND_COMPLEX_TYPES defaults to true; set it to false so that 'select *' returns the
+# deeply nested scalar item instead of expanding the complex-typed columns.
+set expand_complex_types=false;
select * from $DATABASE.int_array_tbl.f t0,
t0.item t1, t1.item t2, t2.item t3, t3.item t4, t4.item t5,
t5.item t6, t6.item t7, t7.item t8, t8.item t9, t9.item t10,
@@ -60,6 +66,9 @@ int
====
---- QUERY
# Test absolute table ref executed with a single scan.
+# EXPAND_COMPLEX_TYPES defaults to true; set it to false so that 'select *' returns the
+# deeply nested scalar item instead of expanding the complex-typed columns.
+set expand_complex_types=false;
select * from $DATABASE.struct_array_tbl.f.
f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.
f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.f0.
@@ -71,6 +80,9 @@ int
====
---- QUERY
# Test relative ref executed with deeply nested subplans.
+# EXPAND_COMPLEX_TYPES defaults to true; set it to false so that 'select *' returns the
+# deeply nested scalar item instead of expanding the complex-typed columns.
+set expand_complex_types=false;
select * from $DATABASE.struct_array_tbl.f t0,
t0.f0 t1, t1.f0 t2, t2.f0 t3, t3.f0 t4, t4.f0 t5,
t5.f0 t6, t6.f0 t7, t7.f0 t8, t8.f0 t9, t9.f0 t10,
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-struct-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-struct-in-select-list.test
index e00500632..63c6f0d64 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-struct-in-select-list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-struct-in-select-list.test
@@ -415,6 +415,9 @@ STRING,STRING
====
---- QUERY
# Checks that "SELECT nested_struct.* ..." omits the nested structs from the output.
+# This behaviour requires EXPAND_COMPLEX_TYPES=false (its default is true), otherwise
+# 'outer_struct.*' would also expand the nested struct fields.
+set expand_complex_types=false;
select id, outer_struct.* from complextypes_nested_structs;
---- RESULTS
1,'somestr1'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test
index 4579fb094..d62637516 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test
@@ -10,6 +10,7 @@ INT, STRING, STRING
====
---- QUERY
# Expanding star on table with array type, and an additional scalar field, without EXPAND_COMPLEX_TYPES
+set EXPAND_COMPLEX_TYPES=false;
select id, * from complextypes_arrays where id = 2
---- RESULTS
2,2
@@ -55,6 +56,7 @@ BIGINT, BIGINT, STRING, STRING, BIGINT, STRING, STRING
====
---- QUERY
# Expanding star on views with array type, and an additional scalar field, without EXPAND_COMPLEX_TYPES
+set EXPAND_COMPLEX_TYPES=false;
select v1.id, *
from complextypes_arrays_only_view v1, complextypes_arrays_only_view v2
where v1.id = v2.id and v1.id = 7
@@ -123,6 +125,7 @@ INT, STRING, STRING, STRING, STRING, INT, STRING, STRING
====
---- QUERY
# Expanding star on tables with array and struct type, without EXPAND_COMPLEX_TYPES
+set EXPAND_COMPLEX_TYPES=false;
select a.*, s.*
from complextypes_arrays a, complextypes_structs s
where a.id = s.id and a.id = 6
@@ -154,6 +157,7 @@ INT, STRING, STRING, STRING, STRING, INT, STRING, STRING, BIGINT, STRING, STRING
====
---- QUERY
# Expanding struct, map and array as separate aliases, without EXPAND_COMPLEX_TYPES
+set EXPAND_COMPLEX_TYPES=false;
select s.*, a.*, m.*
from complextypes_structs s, complextypes_arrays a, complextypes_maps_view m
where s.id = a.id and s.id = m.id and s.id = 3
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test
index f0f74a28f..1685af6f7 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-subplan.test
@@ -111,6 +111,10 @@ bigint,string,bigint,string
====
---- QUERY
# Union inside a subplan.
+# EXPAND_COMPLEX_TYPES defaults to true; set it to false here because 'select *' on the
+# c_orders collection would otherwise expand the nested o_lineitems array and set
+# operations don't support STRUCT-typed columns.
+set expand_complex_types=false;
select c_custkey, c_mktsegment, o_orderkey, o_orderdate
from customer c,
(select * from c.c_orders where o_orderkey % 2 = 0
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test
index 764742a26..7c652243e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch-mem-limit.test
@@ -5,6 +5,7 @@
# mem_limit is tuned for a 3-node HDFS minicluster.
set num_scanner_threads=1;
set mem_limit=1g;
+set expand_complex_types=false;
select *
from customer c, c.c_orders o, o.o_lineitems
order by l_partkey desc, l_suppkey desc, l_linenumber desc, c_custkey
@@ -24,6 +25,7 @@ bigint,string,string,smallint,string,decimal,string,string,bigint,string,decimal
# mem_limit is tuned for a 3-node HDFS minicluster.
set num_scanner_threads=1;
set mem_limit=500m;
+set expand_complex_types=false;
select *
from customer c, c.c_orders o, o.o_lineitems
where c_phone='20-968-632-1388' and l_partkey = 127499
diff --git a/testdata/workloads/functional-query/queries/QueryTest/paimon-query.test b/testdata/workloads/functional-query/queries/QueryTest/paimon-query.test
index 075cfb8ae..4e326f3fc 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/paimon-query.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/paimon-query.test
@@ -137,6 +137,10 @@ select * from functional_parquet.paimon_decimal_tbl where d3 >= 123.45678900 and
DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
====
---- QUERY
+# EXPAND_COMPLEX_TYPES defaults to true; set it to false here because 'select *' would
+# otherwise expand the struct_val column, which the Paimon scanner does not support
+# querying (see the STRUCT cases below).
+set expand_complex_types=false;
SELECT * FROM functional_parquet.alltypes_structs_paimon WHERE year=2009 AND month=1 AND id > 300
---- RESULTS
301,2009,1
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
index fe6ab18d4..dbad2fb2d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
@@ -18,16 +18,16 @@ BIGINT
# Test queries on complex types table.
select * from complextypestbl
---- RESULTS
-100
-200
-300
-400
-500
-600
-700
-800
+100,'[1,2,3]','[[1,2],[3,4]]','{"k1":1,"k2":100}','[{"k1":1}]','{"a":1,"b":[1],"c":{"d":[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]},"g":{"foo":{"h":{"i":[1.1]}}}}'
+200,'[null,1,2,null,3,null]','[[null,1,2,null],[3,null,4],[],null]','{"k1":2,"k2":null}','[{"k3":null,"k1":1},null,{}]','{"a":null,"b":[null],"c":{"d":[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]},"g":{"g1":{"h":{"i":[2.2,null]}},"g2":{"h":{"i":[]}},"g3":null,"g4":{"h":{"i":null}},"g5":{"h":null}}}'
+300,'[]','[null]','{}','[null,null]','{"a":null,"b":null,"c":{"d":[]},"g":{}}'
+400,'NULL','[]','{}','[]','{"a":null,"b":null,"c":{"d":null},"g":null}'
+500,'NULL','NULL','{}','NULL','{"a":null,"b":null,"c":null,"g":{"foo":{"h":{"i":[2.2,3.3]}}}}'
+600,'NULL','NULL','NULL','NULL','NULL'
+700,'NULL','[null,[5,6]]','{"k1":null,"k3":null}','NULL','{"a":7,"b":[2,3,null],"c":{"d":[[],[null],null]},"g":null}'
+800,'[-1]','[[-1,-2],[]]','{"k1":-1}','[{},{"k1":1},{},{}]','{"a":-1,"b":[-1],"c":{"d":[[{"e":-1,"f":"nonnullable"}]]},"g":{}}'
---- TYPES
-BIGINT
+BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Test resolving nested column of the masked table.
@@ -48,31 +48,31 @@ BIGINT,INT
# Test resolving nested columns in expanding star expression.
select id, nested_struct.* from complextypestbl
---- RESULTS
-100,1
-200,NULL
-300,NULL
-400,NULL
-500,NULL
-600,NULL
-700,7
-800,-1
+800,-1,'[-1]','{"d":[[{"e":-1,"f":"nonnullable"}]]}','{}'
+100,1,'[1]','{"d":[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]}','{"foo":{"h":{"i":[1.1]}}}'
+200,NULL,'[null]','{"d":[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]}','{"g1":{"h":{"i":[2.2,null]}},"g2":{"h":{"i":[]}},"g3":null,"g4":{"h":{"i":null}},"g5":{"h":null}}'
+300,NULL,'NULL','{"d":[]}','{}'
+400,NULL,'NULL','{"d":null}','NULL'
+500,NULL,'NULL','NULL','{"foo":{"h":{"i":[2.2,3.3]}}}'
+600,NULL,'NULL','NULL','NULL'
+700,7,'[2,3,null]','{"d":[[],[null],null]}','NULL'
---- TYPES
-BIGINT,INT
+BIGINT, INT, STRING, STRING, STRING
====
---- QUERY
# Test resolving nested columns in expanding star expression.
select nested_struct.* from complextypestbl
---- RESULTS
-1
-NULL
-NULL
-NULL
-NULL
-NULL
-7
--1
----- TYPES
-INT
+-1,'[-1]','{"d":[[{"e":-1,"f":"nonnullable"}]]}','{}'
+1,'[1]','{"d":[[{"e":10,"f":"aaa"},{"e":-10,"f":"bbb"}],[{"e":11,"f":"c"}]]}','{"foo":{"h":{"i":[1.1]}}}'
+NULL,'[null]','{"d":[[{"e":null,"f":null},{"e":10,"f":"aaa"},{"e":null,"f":null},{"e":-10,"f":"bbb"},{"e":null,"f":null}],[{"e":11,"f":"c"},null],[],null]}','{"g1":{"h":{"i":[2.2,null]}},"g2":{"h":{"i":[]}},"g3":null,"g4":{"h":{"i":null}},"g5":{"h":null}}'
+NULL,'NULL','{"d":[]}','{}'
+NULL,'NULL','{"d":null}','NULL'
+NULL,'NULL','NULL','{"foo":{"h":{"i":[2.2,3.3]}}}'
+NULL,'NULL','NULL','NULL'
+7,'[2,3,null]','{"d":[[],[null],null]}','NULL'
+---- TYPES
+INT, STRING, STRING, STRING
====
---- QUERY
# Test resolving nested columns in expanding star expression.
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_row_filtering.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_row_filtering.test
index 85255be83..41322260f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_row_filtering.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_row_filtering.test
@@ -560,6 +560,9 @@ BIGINT
====
---- QUERY
# Row-filtering policy keeps rows with "nested_struct.a is not NULL"
+# EXPAND_COMPLEX_TYPES defaults to true; set it to false here so that 'nested_struct.*'
+# expands only the scalar field 'a' and the test stays focused on row filtering.
+set expand_complex_types=false;
select id, nested_struct.* from functional_parquet.complextypestbl
---- RESULTS
1,1
diff --git a/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-parquet.test b/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-parquet.test
index bf54e6e5c..10cf75f77 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-parquet.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-parquet.test
@@ -251,16 +251,16 @@ BIGINT, INT, STRING, INT, INT, INT, INT, INT, STRING, STRING, STRING, STRING, IN
---- QUERY
select * from customer_nested_multiblock_multipage where c_custkey = 300;
---- RESULTS
-300,'Customer#000000300','I0fJfo60DRqQ',7,'17-165-193-5964',8084.92,'AUTOMOBILE','p fluffily among the slyly express grouches. furiously express instruct'
+300,'Customer#000000300','I0fJfo60DRqQ',7,'17-165-193-5964',8084.92,'AUTOMOBILE','p fluffily among the slyly express grouches. furiously express instruct','NULL'
---- TYPES
-BIGINT, STRING, STRING, SMALLINT, STRING, DECIMAL, STRING, STRING
+BIGINT, STRING, STRING, SMALLINT, STRING, DECIMAL, STRING, STRING, STRING
====
---- QUERY
select * from customer_nested_multiblock_multipage where c_name = 'Customer#000000295';
---- RESULTS
-295,'Customer#000000295','mk649IH6njR14woTVZ1cxtlNs URxBHD5o5z2',0,'10-340-773-4322',9497.89,'HOUSEHOLD','play according to the quickly ironic instructions-- unusual, bol'
+295,'Customer#000000295','mk649IH6njR14woTVZ1cxtlNs URxBHD5o5z2',0,'10-340-773-4322',9497.89,'HOUSEHOLD','play according to the quickly ironic instructions-- unusual, bol','[{"o_orderkey":1340643,"o_orderstatus":"O","o_totalprice":176020.28,"o_orderdate":"1997-08-14","o_orderpriority":"2-HIGH","o_clerk":"Clerk#000000970","o_shippriority":0,"o_comment":"quickly special platelets cajo","o_lineitems":[{"l_partkey":182078,"l_suppkey":4597,"l_linenumber":1,"l_quantity":1.00,"l_extendedprice": [...]
---- TYPES
-BIGINT, STRING, STRING, SMALLINT, STRING, DECIMAL, STRING, STRING
+BIGINT, STRING, STRING, SMALLINT, STRING, DECIMAL, STRING, STRING, STRING
====
---- QUERY
select file__position, l_shipdate from customer_nested_multiblock_multipage c, c.c_orders.o_lineitems where l_shipdate='1998-11-26';
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 3bbb6b05a..42fbafa01 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -240,6 +240,10 @@ class TestLineage(CustomClusterTestSuite):
impalad_args="--lineage_event_log_dir={" + LINEAGE + "}",
tmp_dir_placeholders=[LINEAGE])
def test_lineage_output(self, vector):
+ # The expected lineage graphs in lineage.test predate EXPAND_COMPLEX_TYPES defaulting
+ # to true. Keep '*' skipping complex-typed columns so the graphs stay valid (and so
+ # queries selecting '*' from text-format tables with complex columns don't error).
+ vector.get_value('exec_option')['expand_complex_types'] = 'false'
try:
self.run_test_case('QueryTest/lineage', vector)
finally:
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index 84a899354..4d401991e 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -2014,8 +2014,12 @@ class TestKuduArray(KuduTestSuite):
"{0}.array_{1}".format(table_name, self._get_name_from_type(item_type))
for item_type in types
])
+ # EXPAND_COMPLEX_TYPES defaults to true; disable it so that 'SELECT *' does not also
+ # expand the table's ARRAY columns. Only the UNNEST-ed items are expected here.
+ no_expand_options = dict(query_options)
+ no_expand_options["expand_complex_types"] = "false"
result = self.execute_query("SELECT * FROM {1}.{2}, UNNEST({0})".format(
- columns, db, table_name), query_options=query_options)
+ columns, db, table_name), query_options=no_expand_options)
for i, result_column in enumerate(zip(*result.tuples())):
if i == 0:
assert result_column == self.EXPECTED_ID_UNNESTED