You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/05/14 20:12:49 UTC
(impala) branch master updated (06629bf0f -> 9f9e36be8)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 06629bf0f IMPALA-12679: Deflake rpc_count check in TestFetch.test_rows_sent_counters
new 44376d859 IMPALA-14937: (addendum) Fix NPE on shuffle expression creation for Iceberg V3 MERGE statements
new 4237d34de IMPALA-14987: test_topic_updates_unblock_legacy_catalog is flaky
new 9f9e36be8 IMPALA-14986: Fix test_cancelled_nodes_in_exec_summary
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/impala/analysis/IcebergMergeImpl.java | 6 ++++
.../queries/QueryTest/iceberg-v3-merge.test | 34 ++++++++++++++++++++++
.../custom_cluster/test_topic_update_frequency.py | 24 +++++++--------
tests/query_test/test_observability.py | 23 ++++++++-------
4 files changed, 64 insertions(+), 23 deletions(-)
(impala) 03/03: IMPALA-14986: Fix test_cancelled_nodes_in_exec_summary
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9f9e36be82fdad632c1fc75b16b152ef19f997df
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Wed May 13 16:40:59 2026 +0200
IMPALA-14986: Fix test_cancelled_nodes_in_exec_summary
Test test_cancelled_nodes_in_exec_summary used hard-coded HDFS. This
patch modifies the test to use the FILESYSTEM_NAME.
Testing
* Executed the test on S3 and on Ozone
Change-Id: I6be78952aa8ba249381440e07213ad1cc124a183
Reviewed-on: http://gerrit.cloudera.org:8080/24304
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Jason Fehr <jf...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/query_test/test_observability.py | 23 ++++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 98c3b3403..d1e190d76 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -30,7 +30,7 @@ from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfFS, SkipIfLocal, SkipIfNotHdfsMinicluster,
SkipIfCalcite)
from tests.common.test_vector import HS2
-from tests.util.filesystem_utils import WAREHOUSE
+from tests.util.filesystem_utils import FILESYSTEM_NAME, WAREHOUSE
from tests.util.parse_util import get_duration_us_from_str
from tests.util.query_profile_util import (verify_profile_event_sequence,
verify_profile_node_lifecycle_events)
@@ -106,7 +106,7 @@ class TestObservability(ImpalaTestSuite):
query = "select count(*) from functional.alltypestiny"
result = self.client.execute(query, fetch_exec_summary=True)
scan_idx = len(result.exec_summary) - 1
- assert result.exec_summary[scan_idx]['operator'] == '00:SCAN HDFS'
+ assert result.exec_summary[scan_idx]['operator'] == '00:SCAN ' + FILESYSTEM_NAME
assert result.exec_summary[scan_idx]['detail'] == 'functional.alltypestiny'
# KUDU table
@@ -178,10 +178,11 @@ class TestObservability(ImpalaTestSuite):
# 00:UNION 30 10
# |--02:SCAN HDFS 0 150.00K tpch.customer, CANCELLED
# 01:SCAN HDFS 3.07K 6.00M tpch.lineitem, CANCELLED
- assert result.exec_summary[4]['operator'] == '02:SCAN HDFS', result.runtime_profile
+ scan_op = 'SCAN ' + FILESYSTEM_NAME
+ assert result.exec_summary[4]['operator'] == '02:' + scan_op, result.runtime_profile
assert result.exec_summary[4]['detail'] == 'tpch.customer, CANCELLED', \
result.runtime_profile
- assert result.exec_summary[5]['operator'] == '01:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[5]['operator'] == '01:' + scan_op, result.runtime_profile
assert result.exec_summary[5]['detail'] == 'tpch.lineitem, CANCELLED', \
result.runtime_profile
@@ -210,13 +211,13 @@ class TestObservability(ImpalaTestSuite):
# 06:EXCHANGE 1 1 UNPARTITIONED
# F00:EXCHANGE SENDER
# 00:SCAN HDFS 3 1 tpch.lineitem a
- assert result.exec_summary[8]['operator'] == '03:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[8]['operator'] == '03:' + scan_op, result.runtime_profile
assert result.exec_summary[8]['detail'] == 'tpch.lineitem, CANCELLED', \
result.runtime_profile
- assert result.exec_summary[9]['operator'] == '02:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[9]['operator'] == '02:' + scan_op, result.runtime_profile
assert result.exec_summary[9]['detail'] == 'tpch.lineitem, CANCELLED', \
result.runtime_profile
- assert result.exec_summary[12]['operator'] == '00:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[12]['operator'] == '00:' + scan_op, result.runtime_profile
assert result.exec_summary[12]['detail'] == 'tpch.lineitem a', result.runtime_profile
# Test on other node types
@@ -241,7 +242,7 @@ class TestObservability(ImpalaTestSuite):
# F00:EXCHANGE SENDER
# 02:SORT 661.05K 6.00M CANCELLED
# 01:SCAN HDFS 6.00M 6.00M tpch.lineitem
- assert result.exec_summary[4]['operator'] == '04:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[4]['operator'] == '04:' + scan_op, result.runtime_profile
assert result.exec_summary[4]['detail'] == 'tpch.customer, CANCELLED', \
result.runtime_profile
assert result.exec_summary[5]['operator'] == '06:EXCHANGE', result.runtime_profile
@@ -254,7 +255,7 @@ class TestObservability(ImpalaTestSuite):
result.runtime_profile
assert result.exec_summary[10]['operator'] == '02:SORT', result.runtime_profile
assert result.exec_summary[10]['detail'] == 'CANCELLED', result.runtime_profile
- assert result.exec_summary[11]['operator'] == '01:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[11]['operator'] == '01:' + scan_op, result.runtime_profile
assert result.exec_summary[11]['detail'] == 'tpch.lineitem', result.runtime_profile
# Test limit on Subplan. As the SubplanNode reaches its limit, it finishes as expected
@@ -289,7 +290,7 @@ class TestObservability(ImpalaTestSuite):
assert result.exec_summary[3]['detail'] == '', result.runtime_profile
assert result.exec_summary[4]['operator'] == '03:UNNEST', result.runtime_profile
assert result.exec_summary[4]['detail'] == 'c.c_orders arr', result.runtime_profile
- assert result.exec_summary[5]['operator'] == '00:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[5]['operator'] == '00:' + scan_op, result.runtime_profile
assert result.exec_summary[5]['detail'] == \
'tpch_nested_parquet.customer c, CANCELLED', result.runtime_profile
@@ -326,7 +327,7 @@ class TestObservability(ImpalaTestSuite):
assert result.exec_summary[7]['operator'] == '04:UNNEST', result.runtime_profile
assert result.exec_summary[7]['detail'] == 'c.c_orders arr, CANCELLED', \
result.runtime_profile
- assert result.exec_summary[8]['operator'] == '01:SCAN HDFS', result.runtime_profile
+ assert result.exec_summary[8]['operator'] == '01:' + scan_op, result.runtime_profile
assert result.exec_summary[8]['detail'] == \
'tpch_nested_orc_def.customer c, CANCELLED', result.runtime_profile
(impala) 02/03: IMPALA-14987: test_topic_updates_unblock_legacy_catalog is flaky
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 4237d34decdd7f2ccf019340502a6499eb325a72
Author: Mihaly Szjatinya <ms...@pm.me>
AuthorDate: Thu May 14 10:58:52 2026 +0200
IMPALA-14987: test_topic_updates_unblock_legacy_catalog is flaky
In recent IMPALA-14801 expected query times were reduced, but apparently
overly so. Doubling all times proportionally.
Change-Id: I4c1f01de08d4d17972579ead96499adcdb9eb038
Reviewed-on: http://gerrit.cloudera.org:8080/24302
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../custom_cluster/test_topic_update_frequency.py | 24 +++++++++++-----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/tests/custom_cluster/test_topic_update_frequency.py b/tests/custom_cluster/test_topic_update_frequency.py
index 715c07750..a5dcc4c6c 100644
--- a/tests/custom_cluster/test_topic_update_frequency.py
+++ b/tests/custom_cluster/test_topic_update_frequency.py
@@ -97,9 +97,9 @@ class TestTopicUpdateFrequency(CustomClusterTestSuite):
# holding the table lock. These debug actions are tuned such that each of the blocking
# queries below take little more than 10 seconds (much slower than fast queries).
debug_action = (
- "catalogd_refresh_hdfs_listing_delay:SLEEP@6|"
- "catalogd_table_recover_delay:SLEEP@2000|"
- "catalogd_update_stats_delay:SLEEP@2000")
+ "catalogd_refresh_hdfs_listing_delay:SLEEP@12|"
+ "catalogd_table_recover_delay:SLEEP@4000|"
+ "catalogd_update_stats_delay:SLEEP@4000")
blocking_query_options = {
"debug_action": debug_action
}
@@ -160,9 +160,9 @@ class TestTopicUpdateFrequency(CustomClusterTestSuite):
self.client.execute("refresh tpcds.store_sales")
self.client.execute("refresh functional.alltypes")
debug_action = (
- "catalogd_refresh_hdfs_listing_delay:SLEEP@6|"
- "catalogd_table_recover_delay:SLEEP@2000|"
- "catalogd_update_stats_delay:SLEEP@2000")
+ "catalogd_refresh_hdfs_listing_delay:SLEEP@12|"
+ "catalogd_table_recover_delay:SLEEP@4000|"
+ "catalogd_update_stats_delay:SLEEP@4000")
blocking_query_options = {
"debug_action": debug_action,
"sync_ddl": "true"
@@ -183,8 +183,8 @@ class TestTopicUpdateFrequency(CustomClusterTestSuite):
def __run_topic_update_test_inner(self, slow_blocking_query, fast_queries,
init_queries, blocking_query_options,
- non_blocking_query_options=None, blocking_query_min_time=2000,
- fast_query_timeout_ms=1000, non_blocking_impalad=0,
+ non_blocking_query_options=None, blocking_query_min_time=4000,
+ fast_query_timeout_ms=2000, non_blocking_impalad=0,
expect_topic_updates_to_block=False):
"""This function runs the slow query in a Impala client and then creates separate
Impala clients to run the fast_queries. It makes sure that the
@@ -252,9 +252,9 @@ class TestTopicUpdateFrequency(CustomClusterTestSuite):
"compute stats functional.alltypes"
]
debug_action = (
- "catalogd_refresh_hdfs_listing_delay:SLEEP@6|"
- "catalogd_table_recover_delay:SLEEP@2000|"
- "catalogd_update_stats_delay:SLEEP@2000")
+ "catalogd_refresh_hdfs_listing_delay:SLEEP@12|"
+ "catalogd_table_recover_delay:SLEEP@4000|"
+ "catalogd_update_stats_delay:SLEEP@4000")
blocking_query_options = {
"debug_action": debug_action,
"sync_ddl": "true"
@@ -406,7 +406,7 @@ class TestTopicUpdateFrequency(CustomClusterTestSuite):
# Debug action delays compute stats operations on the Iceberg table
# catalogd_update_stats_delay works for both HdfsTable and IcebergTable
# since IcebergTable delegates to its internal HdfsTable
- debug_action = "catalogd_update_stats_delay:SLEEP@2000"
+ debug_action = "catalogd_update_stats_delay:SLEEP@4000"
blocking_query_options = {
"debug_action": debug_action,
}
(impala) 01/03: IMPALA-14937: (addendum) Fix NPE on shuffle expression creation for Iceberg V3 MERGE statements
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 44376d85925fe04a5e78b184f52183d0ea24fd49
Author: Peter Rozsa <pr...@cloudera.com>
AuthorDate: Wed May 13 17:20:59 2026 +0200
IMPALA-14937: (addendum) Fix NPE on shuffle expression creation for Iceberg V3 MERGE statements
isPartitioned() checks all historical partition specs, not just the
current default spec. A table that was created with a non-VOID partition
spec and then evolved to all-VOID still returns true from
isPartitioned().
The fix guards the partitioned V3 shuffle path with an additional
!targetPartitionExpressions_.isEmpty() check so that a table whose
current spec has no active partition fields uses PARTITION__SPEC__ID
and ICEBERG__PARTITION__SERIALIZED for shuffling.
Testing:
- Added a new test case to iceberg-v3-merge.test.
Change-Id: Ia2b0b86deac12e2566d5c24561e120e30b3b6739
Reviewed-on: http://gerrit.cloudera.org:8080/24303
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/analysis/IcebergMergeImpl.java | 6 ++++
.../queries/QueryTest/iceberg-v3-merge.test | 34 ++++++++++++++++++++++
2 files changed, 40 insertions(+)
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java b/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java
index 828ae548a..7d9c732eb 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java
@@ -275,6 +275,11 @@ public class IcebergMergeImpl implements MergeImpl {
private List<Expr> buildShuffleExprs(Analyzer analyzer) throws AnalysisException {
if (icebergTable_.isPartitioned()) {
if (icebergTable_.getFormatVersion() >= IcebergUtil.FORMAT_VERSION_3) {
+ // targetPartitionExpressions_ may be empty when isPartitioned() returns true
+ // based on a historical spec while the current default spec is all-VOID.
+ if (targetPartitionExpressions_.isEmpty()) {
+ return targetPartitionMetaExpressions_;
+ }
return buildV3PartitionedShuffleExprs(analyzer);
}
return targetPartitionExpressions_;
@@ -293,6 +298,7 @@ public class IcebergMergeImpl implements MergeImpl {
* ICEBERG__PARTITION__SERIALIZED]
* murmur_hash takes a single argument, so N partition transform expressions are
* hashed individually and XOR'd together into one BIGINT.
+ * Precondition: targetPartitionExpressions_ is non-empty.
*/
private List<Expr> buildV3PartitionedShuffleExprs(Analyzer analyzer)
throws AnalysisException {
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-merge.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-merge.test
index 6c81b1b14..2ff444114 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-merge.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-merge.test
@@ -1265,3 +1265,37 @@ FROM s21_target ORDER BY id;
---- TYPES
INT,STRING,INT,BIGINT
====
+---- QUERY
+# S22: MERGE on a V3 table whose partition spec was evolved to all-VOID.
+CREATE TABLE s22_target (id INT, val STRING)
+PARTITIONED BY SPEC (id)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+
+CREATE TABLE s22_source (id INT, val STRING)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+
+INSERT INTO s22_target VALUES (1,'one'),(2,'two'),(3,'three');
+INSERT INTO s22_source VALUES (2,'TWO'),(4,'four');
+====
+---- QUERY
+-- Evolve to all-VOID: current default spec has no active partition fields.
+ALTER TABLE s22_target SET PARTITION SPEC (void(id));
+====
+---- QUERY
+MERGE INTO s22_target USING s22_source ON s22_target.id = s22_source.id
+WHEN MATCHED THEN UPDATE SET s22_target.val = s22_source.val
+WHEN NOT MATCHED THEN INSERT VALUES (s22_source.id, s22_source.val);
+---- DML_RESULTS: s22_target
+1,'one'
+2,'TWO'
+3,'three'
+4,'four'
+---- TYPES
+INT,STRING
+---- RUNTIME_PROFILE
+# After evolving to all-VOID the plan must shuffle on PARTITION__SPEC__ID and
+# ICEBERG__PARTITION__SERIALIZED.
+row_regex: .*EXCHANGE \[HASH\(.*partition__spec__id.*iceberg__partition__serialized.*\).*
+====