You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2025/06/20 22:04:25 UTC
(impala) branch master updated (060432595 -> 892b33f35)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 060432595 IMPALA-13901: Make histogram/sample/group_concat ineligible for tuple caching
new 1a381b6ae IMPALA-14142: Fix TestIcebergV2Table.test_compute_stats_table_sampling
new 892b33f35 IMPALA-14154: IllegalStateException with Iceberg table with DELETE
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/impala/planner/IcebergScanPlanner.java | 8 +
.../iceberg-v2-compute-stats-table-sampling.test | 234 ---------------------
.../iceberg-v2-read-position-deletes.test | 21 ++
tests/metadata/test_stats_extrapolation.py | 66 +++++-
tests/query_test/test_iceberg.py | 8 -
5 files changed, 84 insertions(+), 253 deletions(-)
delete mode 100644 testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-compute-stats-table-sampling.test
(impala) 02/02: IMPALA-14154: IllegalStateException with Iceberg table with DELETE
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 892b33f35d1cab01d91d9b5eaac079ec8ba236bb
Author: Daniel Becker <db...@apache.org>
AuthorDate: Tue Jun 17 15:11:09 2025 +0200
IMPALA-14154: IllegalStateException with Iceberg table with DELETE
Planning a query on an Iceberg table runs on IllegalStateException in
the following case:
- the table has delete files for each data file (i.e. no data file
without deletes)
AND
- there is an anti-join on top of the Iceberg delete operation
(IcebergDeleteNode or HashJoinNode).
The exception is thrown by a Preconditions check in
SingleNodePlanner.createJoinNode() because there is no null-matching EQ
operator. This happens because the conjunct that should be there is
discarded, ultimately in Analyzer.canEvalAntiJoinedConjunct(). The
reason is that the tuple ids passed to that function include the delete
part of the Iceberg scan, but the conjunct (correctly) only refers to
the tuple of the data files.
Note that this does not happen if we have two regular anti-joins on top
of each other because in that case the getTblRefIds() method of the
bottom anti-join returns a single TableRefId, the one corresponding to
the (inline) view containing it, not the two TableRefIds corresponding
to its two child nodes. The conjunct references this TupleId, so
Analyzer.canEvalAntiJoinedConjunct() returns true. This is not the case
with Iceberg delete operations because there is no (inline) view
involved.
This commit solves the issue by setting the TableRefIds of the node
corresponding to the Iceberg delete operation (IcebergDeleteNode or
HashJoinNode) to only the table ref that corresponds to the data files,
not the delete files.
Testing:
- added a test in iceberg-v2-read-position-deletes.test that
reproduces the issue.
Change-Id: If2c03fe3da44dc0516ebdf32430416a1059d37b2
Reviewed-on: http://gerrit.cloudera.org:8080/23051
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/planner/IcebergScanPlanner.java | 8 ++++++++
.../QueryTest/iceberg-v2-read-position-deletes.test | 21 +++++++++++++++++++++
2 files changed, 29 insertions(+)
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
index ee1e8de5e..b7ef9b6d6 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
@@ -340,6 +340,14 @@ public class IcebergScanPlanner {
joinNode.setId(ctx_.getNextNodeId());
joinNode.init(analyzer_);
joinNode.setIsDeleteRowsJoin();
+
+ // The output of this node only contains the tuple corresponding to 'dataScanNode',
+ // not that of 'deleteScanNode'. Conjuncts above this node, e.g. in another join, will
+ // only reference that tuple, so we should only include the table ref of
+ // 'dataScanNode' here.
+ // See IMPALA-14154.
+ joinNode.setTblRefIds(dataScanNode.getTblRefIds());
+
return joinNode;
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-position-deletes.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-position-deletes.test
index a0f888d3d..067eddbaf 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-position-deletes.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-position-deletes.test
@@ -575,3 +575,24 @@ bigint
aggregation(SUM, NumRowGroups): 0
aggregation(SUM, NumFileMetadataRead): 1
====
+---- QUERY
+# Regression test for IMPALA-14154.
+# To reproduce the bug, we need a table where there is no data file without a delete file.
+# The metadata table query ensures that.
+select content from functional_parquet.iceberg_v2_delete_positional.`files`;
+---- RESULTS
+0
+1
+---- TYPES
+INT
+====
+---- QUERY
+# Regression test for IMPALA-14154.
+select `data`
+from functional_parquet.iceberg_v2_delete_positional
+where `data` not in (select min(`data`) from functional_parquet.iceberg_v2_delete_positional);
+---- RESULTS
+'c'
+---- TYPES
+STRING
+====
(impala) 01/02: IMPALA-14142: Fix TestIcebergV2Table.test_compute_stats_table_sampling
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1a381b6aea2d19d738119bd0e1c457e13a0d2763
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Wed Jun 18 15:30:32 2025 +0200
IMPALA-14142: Fix TestIcebergV2Table.test_compute_stats_table_sampling
TestIcebergV2Table.test_compute_stats_table_sampling was failing in
ARM release builds. However, COMPUTE STATS with TABLESAMPLE is
inherently non-deterministic due to its use of SAMPLED_NDV().
This patch completely rewrites the tests and moves them to
test_stats_extrapolation.py to test Iceberg tables similarly to
legacy tables.
'diff_perc' argument of appx_equals() method was also updated in
the tests, as with the previous value (1.0) it only reported errors
for negative estimates.
Change-Id: I98b07b156aad300827c9e1b7970b8dfacfc6d251
Reviewed-on: http://gerrit.cloudera.org:8080/23044
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../iceberg-v2-compute-stats-table-sampling.test | 234 ---------------------
tests/metadata/test_stats_extrapolation.py | 66 +++++-
tests/query_test/test_iceberg.py | 8 -
3 files changed, 55 insertions(+), 253 deletions(-)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-compute-stats-table-sampling.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-compute-stats-table-sampling.test
deleted file mode 100644
index 19e81836a..000000000
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-compute-stats-table-sampling.test
+++ /dev/null
@@ -1,234 +0,0 @@
-====
----- QUERY
-DROP STATS iceberg_non_partitioned;
-COMPUTE STATS iceberg_non_partitioned tablesample system(10) repeatable(1234);
----- RESULTS
-'Updated 1 partition(s) and 4 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_non_partitioned;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-20,20,'22.90KB','NOT CACHED','NOT CACHED','PARQUET','false','$NAMENODE/test-warehouse/iceberg_test/iceberg_non_partitioned','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_non_partitioned;
----- RESULTS
-'id','INT',3,0,4,4,-1,-1
-'user','STRING',2,0,4,4,-1,-1
-'action','STRING',2,0,5,4.333333492279053,-1,-1
-'event_time','TIMESTAMP',2,0,16,16,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_non_partitioned;
-COMPUTE STATS iceberg_non_partitioned tablesample system(10) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 4 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_non_partitioned;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-20,20,'22.90KB','NOT CACHED','NOT CACHED','PARQUET','false','$NAMENODE/test-warehouse/iceberg_test/iceberg_non_partitioned','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_non_partitioned;
----- RESULTS
-'id','INT',2,0,4,4,-1,-1
-'user','STRING',2,0,4,4,-1,-1
-'action','STRING',2,0,8,6.5,-1,-1
-'event_time','TIMESTAMP',2,0,16,16,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_partitioned;
-COMPUTE STATS iceberg_partitioned tablesample system(10) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 4 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_partitioned;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-20,20,'22.90KB','NOT CACHED','NOT CACHED','PARQUET','false','$NAMENODE/test-warehouse/iceberg_test/iceberg_partitioned','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_partitioned;
----- RESULTS
-'id','INT',3,0,4,4,-1,-1
-'user','STRING',3,0,4,4,-1,-1
-'action','STRING',3,0,8,5.666666507720947,-1,-1
-'event_time','TIMESTAMP',3,0,16,16,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_v2_delete_equality_partitioned;
-COMPUTE STATS iceberg_v2_delete_equality_partitioned tablesample system(10) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 3 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_v2_delete_equality_partitioned;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-3,6,'4.81KB','NOT CACHED','NOT CACHED','PARQUET','false','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_v2_delete_equality_partitioned;
----- RESULTS
-'i','INT',2,0,4,4,-1,-1
-'s','STRING',2,0,4,4,-1,-1
-'d','DATE',1,0,4,4,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_v2_delete_equality_partitioned;
-COMPUTE STATS iceberg_v2_delete_equality_partitioned tablesample system(10) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 3 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_v2_delete_equality_partitioned;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-3,6,'4.81KB','NOT CACHED','NOT CACHED','PARQUET','false','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_v2_delete_equality_partitioned;
----- RESULTS
-'i','INT',2,0,4,4,-1,-1
-'s','STRING',2,0,4,4,-1,-1
-'d','DATE',1,0,4,4,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_v2_positional_not_all_data_files_have_delete_files;
-COMPUTE STATS iceberg_v2_positional_not_all_data_files_have_delete_files tablesample system(30) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 2 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_v2_positional_not_all_data_files_have_delete_files;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-1,6,'7.77KB','NOT CACHED','NOT CACHED','PARQUET','false','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_positional_not_all_data_files_have_delete_files','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_v2_positional_not_all_data_files_have_delete_files;
----- RESULTS
-'i','INT',1,0,4,4,-1,-1
-'s','STRING',1,0,1,1,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc;
-COMPUTE STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc tablesample system(30) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 2 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-1,6,'3.97KB','NOT CACHED','NOT CACHED','ORC','false','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_positional_not_all_data_files_have_delete_files_orc','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc;
----- RESULTS
-'i','INT',1,0,4,4,-1,-1
-'s','STRING',1,0,1,1,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_v2_positional_not_all_data_files_have_delete_files;
-COMPUTE STATS iceberg_v2_positional_not_all_data_files_have_delete_files tablesample system(50) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 2 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_v2_positional_not_all_data_files_have_delete_files;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-4,6,'7.77KB','NOT CACHED','NOT CACHED','PARQUET','false','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_positional_not_all_data_files_have_delete_files','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_v2_positional_not_all_data_files_have_delete_files;
----- RESULTS
-'i','INT',4,0,4,4,-1,-1
-'s','STRING',4,0,1,1,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
----- QUERY
-DROP STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc;
-COMPUTE STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc tablesample system(50) repeatable(1111);
----- RESULTS
-'Updated 1 partition(s) and 2 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-SHOW TABLE STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc;
----- LABELS
-#ROWS, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental stats, Location, EC Policy
----- RESULTS: VERIFY_IS_EQUAL
-4,6,'3.97KB','NOT CACHED','NOT CACHED','ORC','false','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_positional_not_all_data_files_have_delete_files_orc','$ERASURECODE_POLICY'
----- TYPES
-BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING,STRING
-====
----- QUERY
-SHOW COLUMN STATS iceberg_v2_positional_not_all_data_files_have_delete_files_orc;
----- RESULTS
-'i','INT',4,0,4,4,-1,-1
-'s','STRING',4,0,1,1,-1,-1
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE, BIGINT, BIGINT
-====
diff --git a/tests/metadata/test_stats_extrapolation.py b/tests/metadata/test_stats_extrapolation.py
index fccba527f..7d762c5d4 100644
--- a/tests/metadata/test_stats_extrapolation.py
+++ b/tests/metadata/test_stats_extrapolation.py
@@ -55,7 +55,7 @@ class TestStatsExtrapolation(ImpalaTestSuite):
# Since our test tables are small, set the minimum sample size to 0 to make sure
# we exercise the sampling code paths.
- self.client.execute("set compute_stats_min_sample_size=0")
+ self.client.set_configuration_option('compute_stats_min_sample_size', '0')
# Test partitioned table.
part_test_tbl = unique_database + ".alltypes"
@@ -120,24 +120,57 @@ class TestStatsExtrapolation(ImpalaTestSuite):
self.client.execute(
"compute stats {0} tablesample system(10)".format(wide_test_tbl))
+ def test_compute_stats_tablesample_iceberg(self, unique_database):
+ self.client.set_configuration_option('compute_stats_min_sample_size', '0')
+
+ ice_tbls = ['iceberg_non_partitioned', 'iceberg_partitioned',
+ 'iceberg_v2_delete_equality_partitioned',
+ 'iceberg_v2_positional_not_all_data_files_have_delete_files',
+ 'iceberg_v2_positional_not_all_data_files_have_delete_files_orc']
+
+ # Array of sampling parameters: [(percentage, seed), ...]
+ sampling_params = [(1, 3), (10, 7), (20, 13), (100, 99)]
+
+ for tbl in ice_tbls:
+ orig_tbl = 'functional_parquet.' + tbl
+ cloned_tbl_base = unique_database + '.' + tbl + "_base"
+ cloned_tbl_sample = unique_database + '.' + tbl + "_sample"
+ self.clone_iceberg_table(orig_tbl, cloned_tbl_base)
+ self.clone_iceberg_table(orig_tbl, cloned_tbl_sample)
+ self.__set_extrapolation_tblprop(cloned_tbl_sample)
+ self.client.execute("compute stats {0}".format(cloned_tbl_base))
+
+ for sampling_param in sampling_params:
+ percentage = sampling_param[0]
+ seed = sampling_param[1]
+ self.__run_sampling_test(
+ cloned_tbl_sample, "", cloned_tbl_base, percentage, seed,
+ is_precise_table_stats=True)
+
+ def clone_iceberg_table(self, src_tbl, dst_tbl):
+ # We are cloning external, non-HiveCatalog tables here.
+ self.client.execute("create table {0} like {1}".format(dst_tbl, src_tbl))
+
def __set_extrapolation_tblprop(self, tbl):
"""Alters the given table to enable stats extrapolation via tblproperty."""
self.client.execute("alter table {0} set "
"tblproperties('impala.enable.stats.extrapolation'='true')".format(tbl))
- def __run_sampling_test(self, tbl, cols, expected_tbl, perc, seed):
+ def __run_sampling_test(self, tbl, cols, expected_tbl, perc, seed,
+ is_precise_table_stats=False):
"""Drops stats on 'tbl' and then runs COMPUTE STATS TABLESAMPLE on 'tbl' with the
given column restriction clause, sampling percent and random seed. Checks that
the resulting table and column stats are reasonably close to those of
- 'expected_tbl'."""
+ 'expected_tbl'. For table formats like ICEBERG the table-level stats are always
+ precise, this can be indicated by the parameter 'is_extrapolated_table_stats'"""
self.client.execute("drop stats {0}".format(tbl))
self.client.execute(
"compute stats {0}{1} tablesample system ({2}) repeatable ({3})".format(
tbl, cols, perc, seed))
- self.__check_table_stats(tbl, expected_tbl)
- self.__check_column_stats(cols, tbl, expected_tbl)
+ self.__check_table_stats(tbl, expected_tbl, perc, is_precise_table_stats)
+ self.__check_column_stats(cols, tbl, perc, expected_tbl)
- def __check_table_stats(self, tbl, expected_tbl):
+ def __check_table_stats(self, tbl, expected_tbl, perc, is_precise=False):
"""Checks that the row counts reported in SHOW TABLE STATS on 'tbl' are within 2x
of those reported for 'expected_tbl'. Assumes that COMPUTE STATS was previously run
on 'expected_table' and that COMPUTE STATS TABLESAMPLE was run on 'tbl'."""
@@ -147,25 +180,27 @@ class TestStatsExtrapolation(ImpalaTestSuite):
assert len(actual.column_labels) == len(expected.column_labels)
col_names = actual.column_labels
rows_col_idx = col_names.index("#ROWS")
- extrap_rows_col_idx = col_names.index("EXTRAP #ROWS")
+ extrap_rows_label = "#ROWS" if is_precise else "EXTRAP #ROWS"
+ extrap_rows_col_idx = col_names.index(extrap_rows_label)
for i in range(0, len(actual.data)):
act_cols = actual.data[i].split("\t")
exp_cols = expected.data[i].split("\t")
assert int(exp_cols[rows_col_idx]) >= 0
# The expected_tbl is expected to have valid extrapolated #rows for every partition.
assert int(act_cols[extrap_rows_col_idx]) >= 0
+ diff_perc = 0.0 if is_precise else self.get_diff_perc(perc)
self.appx_equals(
- int(act_cols[extrap_rows_col_idx]), int(exp_cols[rows_col_idx]), 1.0)
+ int(act_cols[extrap_rows_col_idx]), int(exp_cols[rows_col_idx]), diff_perc)
# Only the table-level row count is stored. The partition row counts
# are extrapolated.
if act_cols[0] == "Total":
self.appx_equals(
- int(act_cols[rows_col_idx]), int(exp_cols[rows_col_idx]), 1.0)
+ int(act_cols[rows_col_idx]), int(exp_cols[rows_col_idx]), diff_perc)
elif len(actual.data) > 1:
# Partition row count is expected to not be set.
assert int(act_cols[rows_col_idx]) == -1
- def __check_column_stats(self, cols, tbl, expected_tbl):
+ def __check_column_stats(self, cols, tbl, perc, expected_tbl):
"""Checks that the NDVs in SHOW COLUMNS STATS on 'tbl' are within 2x of those
reported for 'expected_tbl'. Assumes that COMPUTE STATS was previously run
on 'expected_table' and that COMPUTE STATS TABLESAMPLE was run on 'tbl'."""
@@ -184,4 +219,13 @@ class TestStatsExtrapolation(ImpalaTestSuite):
# caller drops the stats before calling COMPUTE STATS.
if cols == "" or act_cols[0] in cols:
assert int(act_cols[ndv_col_idx]) >= 0
- self.appx_equals(int(act_cols[ndv_col_idx]), int(exp_cols[ndv_col_idx]), 1.0)
+ self.appx_equals(int(act_cols[ndv_col_idx]), int(exp_cols[ndv_col_idx]),
+ self.get_diff_perc(perc))
+
+ def get_diff_perc(self, percentage):
+ # Calculates the the parameter 'diff_perc' we give to 'appx_equals()'.
+ # 'diff_perc' should be proportional to the sample percentage. We use percentage * 0.9
+ # to give enough margin for misestimation.
+ assert 0 <= percentage <= 100
+ fraction = percentage / 100.0
+ return 1.0 - fraction * 0.9
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index 8515fb910..141808c67 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -1545,14 +1545,6 @@ class TestIcebergV2Table(IcebergTestSuite):
self.run_test_case('QueryTest/iceberg-v2-read-position-deletes-stats', vector)
self.run_test_case('QueryTest/iceberg-v2-read-position-deletes-orc-stats', vector)
- @SkipIfDockerizedCluster.internal_hostname
- @SkipIf.hardcoded_uris
- @pytest.mark.execute_serially
- def test_compute_stats_table_sampling(self, vector):
- """Tests COMPUTE STATS with table sampling."""
- vector.get_value('exec_option')['COMPUTE_STATS_MIN_SAMPLE_SIZE'] = 0
- self.run_test_case('QueryTest/iceberg-v2-compute-stats-table-sampling', vector)
-
@SkipIfFS.hive
def test_read_mixed_format_position_deletes(self, vector, unique_database):
self.run_test_case('QueryTest/iceberg-mixed-format-position-deletes',