You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by wz...@apache.org on 2024/02/07 23:22:24 UTC
(impala) branch master updated (68e61c1aa -> 75947e254)
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 68e61c1aa IMPALA-12787: Concurrent DELETE and UPDATE operations on Iceberg tables can be problematic
new fc74ca672 IMPALA-12378: Auto Ship JDBC Data Source
new 75947e254 IMPALA-12072: Include snapshot id of Iceberg tables in query plan / profile
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/external-data-source-executor.cc | 6 +-
bin/impala-config.sh | 1 +
bin/rat_exclude_files.txt | 2 +-
fe/pom.xml | 6 +
.../org/apache/impala/compat/MetastoreShim.java | 6 +-
fe/src/main/cup/sql-parser.cup | 4 +-
.../apache/impala/analysis/CreateDataSrcStmt.java | 11 +-
.../impala/analysis/CreateTableDataSrcStmt.java | 8 +-
.../extdatasource/ExternalDataSourceExecutor.java | 19 +-
.../impala/extdatasource/jdbc/JdbcDataSource.java | 0
.../org/apache/impala/extdatasource/jdbc/README.md | 0
.../extdatasource/jdbc/conf/DatabaseType.java | 0
.../extdatasource/jdbc/conf/JdbcStorageConfig.java | 0
.../jdbc/conf/JdbcStorageConfigManager.java | 0
.../jdbc/dao/DB2DatabaseAccessor.java | 0
.../extdatasource/jdbc/dao/DatabaseAccessor.java | 0
.../jdbc/dao/DatabaseAccessorFactory.java | 0
.../jdbc/dao/GenericJdbcDatabaseAccessor.java | 7 +-
.../jdbc/dao/ImpalaDatabaseAccessor.java | 0
.../extdatasource/jdbc/dao/JdbcRecordIterator.java | 0
.../jdbc/dao/JethroDatabaseAccessor.java | 0
.../jdbc/dao/MsSqlDatabaseAccessor.java | 0
.../jdbc/dao/MySqlDatabaseAccessor.java | 0
.../jdbc/dao/OracleDatabaseAccessor.java | 0
.../jdbc/dao/PostgresDatabaseAccessor.java | 0
.../exception/JdbcDatabaseAccessException.java | 0
.../jdbc/util/QueryConditionUtil.java | 0
.../apache/impala/planner/DataSourceScanNode.java | 37 ++--
.../org/apache/impala/planner/IcebergScanNode.java | 21 ++-
.../apache/impala/planner/IcebergScanPlanner.java | 17 +-
.../java/org/apache/impala/util/IcebergUtil.java | 6 +
.../org/apache/impala/analysis/AnalyzeDDLTest.java | 1 +
.../org/apache/impala/analysis/ParserTest.java | 3 +-
.../apache/impala/customcluster/LdapHS2Test.java | 3 +-
.../org/apache/impala/planner/PlannerTest.java | 19 +-
.../org/apache/impala/planner/PlannerTestBase.java | 9 +-
.../java/org/apache/impala/testutil/TestUtils.java | 3 +
java/pom.xml | 1 +
testdata/bin/copy-ext-data-sources.sh | 7 -
.../iceberg-predicates-disabled-subsetting.test | 5 +-
.../queries/PlannerTest/iceberg-predicates.test | 24 ++-
.../queries/PlannerTest/iceberg-v2-delete.test | 42 +++++
.../PlannerTest/iceberg-v2-tables-hash-join.test | 116 +++++++++++++
.../queries/PlannerTest/iceberg-v2-tables.test | 192 ++++++++++++++++++++-
.../queries/PlannerTest/iceberg-v2-update.test | 42 +++++
.../queries/PlannerTest/insert-sort-by-zorder.test | 2 +
.../queries/PlannerTest/tablesample.test | 11 +-
.../impala-ext-jdbc-tables-predicates.test | 3 +-
.../queries/QueryTest/impala-ext-jdbc-tables.test | 3 +-
.../QueryTest/jdbc-data-source-with-keystore.test | 3 +-
.../queries/QueryTest/jdbc-data-source.test | 3 +-
.../queries/QueryTest/mysql-ext-jdbc-tables.test | 3 +-
tests/query_test/test_iceberg.py | 25 +++
53 files changed, 585 insertions(+), 86 deletions(-)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/JdbcDataSource.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/README.md (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/conf/DatabaseType.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfig.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DB2DatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessorFactory.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/GenericJdbcDatabaseAccessor.java (98%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/ImpalaDatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JdbcRecordIterator.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JethroDatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MsSqlDatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MySqlDatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/OracleDatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/dao/PostgresDatabaseAccessor.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/exception/JdbcDatabaseAccessException.java (100%)
rename {java/ext-data-source/jdbc => fe}/src/main/java/org/apache/impala/extdatasource/jdbc/util/QueryConditionUtil.java (100%)
(impala) 02/02: IMPALA-12072: Include snapshot id of Iceberg tables in query plan / profile
Posted by wz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 75947e254a800fbc168b054d488d529a907c4e63
Author: Gergely Fürnstáhl <gf...@cloudera.com>
AuthorDate: Mon Jul 17 14:23:25 2023 +0200
IMPALA-12072: Include snapshot id of Iceberg tables in query plan / profile
Including the snapshot id of Iceberg tables for the Iceberg SCAN
operators can be useful to enable replayable queries. Replayable
queries are useful, so we can better investigate performance
problems / bugs.
Testing:
- Updated planner tests, added e2e test for time travel
Change-Id: Iee0b4967429ea733729ad8e44df32e3b24b88525
Reviewed-on: http://gerrit.cloudera.org:8080/20204
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/planner/IcebergScanNode.java | 21 ++-
.../apache/impala/planner/IcebergScanPlanner.java | 17 +-
.../java/org/apache/impala/util/IcebergUtil.java | 6 +
.../org/apache/impala/planner/PlannerTest.java | 19 +-
.../org/apache/impala/planner/PlannerTestBase.java | 9 +-
.../java/org/apache/impala/testutil/TestUtils.java | 3 +
.../iceberg-predicates-disabled-subsetting.test | 5 +-
.../queries/PlannerTest/iceberg-predicates.test | 24 ++-
.../queries/PlannerTest/iceberg-v2-delete.test | 42 +++++
.../PlannerTest/iceberg-v2-tables-hash-join.test | 116 +++++++++++++
.../queries/PlannerTest/iceberg-v2-tables.test | 192 ++++++++++++++++++++-
.../queries/PlannerTest/iceberg-v2-update.test | 42 +++++
.../queries/PlannerTest/insert-sort-by-zorder.test | 2 +
.../queries/PlannerTest/tablesample.test | 11 +-
tests/query_test/test_iceberg.py | 25 +++
15 files changed, 506 insertions(+), 28 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
index a5abefea0..90cce6edb 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
@@ -70,6 +70,9 @@ public class IcebergScanNode extends HdfsScanNode {
// already applied them and they won't filter any further rows.
private List<Expr> skippedConjuncts_;
+ // The Iceberg snapshot id used for this scan.
+ private final long snapshotId_;
+
// This member is set when this scan node is the left child of an IcebergDeleteNode or
// in other words when this scan node reads data files that have delete files
// associated. Holds the scan node ID of the right child of the IcebergDeleteNode
@@ -78,15 +81,16 @@ public class IcebergScanNode extends HdfsScanNode {
public IcebergScanNode(PlanNodeId id, TableRef tblRef, List<Expr> conjuncts,
MultiAggregateInfo aggInfo, List<FileDescriptor> fileDescs,
- List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts)
+ List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts, long snapshotId)
throws ImpalaRuntimeException {
this(id, tblRef, conjuncts, aggInfo, fileDescs, nonIdentityConjuncts,
- skippedConjuncts, null);
+ skippedConjuncts, null, snapshotId);
}
public IcebergScanNode(PlanNodeId id, TableRef tblRef, List<Expr> conjuncts,
MultiAggregateInfo aggInfo, List<FileDescriptor> fileDescs,
- List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts, PlanNodeId deleteId)
+ List<Expr> nonIdentityConjuncts, List<Expr> skippedConjuncts, PlanNodeId deleteId,
+ long snapshotId)
throws ImpalaRuntimeException {
super(id, tblRef.getDesc(), conjuncts,
getIcebergPartition(((FeIcebergTable)tblRef.getTable()).getFeFsTable()), tblRef,
@@ -102,6 +106,7 @@ public class IcebergScanNode extends HdfsScanNode {
filesAreSorted_ = true;
}
nonIdentityConjuncts_ = nonIdentityConjuncts;
+ snapshotId_ = snapshotId;
//TODO IMPALA-11577: optimize file format counting
boolean hasParquet = false;
boolean hasOrc = false;
@@ -263,10 +268,14 @@ public class IcebergScanNode extends HdfsScanNode {
@Override
protected String getDerivedExplainString(
String indentPrefix, TExplainLevel detailLevel) {
+ StringBuilder output = new StringBuilder();
+ output.append(
+ indentPrefix + "Iceberg snapshot id: " + String.valueOf(snapshotId_) + "\n");
if (!skippedConjuncts_.isEmpty()) {
- return indentPrefix + String.format("skipped Iceberg predicates: %s\n",
- Expr.getExplainString(skippedConjuncts_, detailLevel));
+ output.append(indentPrefix +
+ String.format("skipped Iceberg predicates: %s\n",
+ Expr.getExplainString(skippedConjuncts_, detailLevel)));
}
- return "";
+ return output.toString();
}
}
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
index 13b0251f4..9b54fbad1 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
@@ -129,6 +129,8 @@ public class IcebergScanPlanner {
private long dataFilesWithDeletesMaxPath_ = 0;
private Set<Long> equalityDeleteSequenceNumbers_ = new HashSet<>();
+ private final long snapshotId_;
+
public IcebergScanPlanner(Analyzer analyzer, PlannerContext ctx,
TableRef iceTblRef, List<Expr> conjuncts, MultiAggregateInfo aggInfo)
throws ImpalaException {
@@ -140,6 +142,7 @@ public class IcebergScanPlanner {
conjuncts_ = conjuncts;
aggInfo_ = aggInfo;
extractIcebergConjuncts();
+ snapshotId_ = IcebergUtil.getSnapshotId(getIceTable(), tblRef_.getTimeTravelSpec());
}
public PlanNode createIcebergScanPlan() throws ImpalaException {
@@ -187,7 +190,7 @@ public class IcebergScanPlanner {
Preconditions.checkState(dataFilesWithDeletes_.isEmpty());
PlanNode ret = new IcebergScanNode(ctx_.getNextNodeId(), tblRef_, conjuncts_,
aggInfo_, dataFilesWithoutDeletes_, nonIdentityConjuncts_,
- getSkippedConjuncts());
+ getSkippedConjuncts(), snapshotId_);
ret.init(analyzer_);
return ret;
}
@@ -211,7 +214,7 @@ public class IcebergScanPlanner {
// can just create a SCAN node for these and do a UNION ALL with the ANTI JOIN.
IcebergScanNode dataScanNode = new IcebergScanNode(
ctx_.getNextNodeId(), tblRef_, conjuncts_, aggInfo_, dataFilesWithoutDeletes_,
- nonIdentityConjuncts_, getSkippedConjuncts());
+ nonIdentityConjuncts_, getSkippedConjuncts(), snapshotId_);
dataScanNode.init(analyzer_);
List<Expr> outputExprs = tblRef_.getDesc().getSlots().stream().map(
SlotRef::new).collect(Collectors.toList());
@@ -246,7 +249,7 @@ public class IcebergScanPlanner {
addDeletePositionSlots(deleteDeltaRef);
IcebergScanNode dataScanNode = new IcebergScanNode(
dataScanNodeId, tblRef_, conjuncts_, aggInfo_, dataFilesWithDeletes_,
- nonIdentityConjuncts_, getSkippedConjuncts(), deleteScanNodeId);
+ nonIdentityConjuncts_, getSkippedConjuncts(), deleteScanNodeId, snapshotId_);
dataScanNode.init(analyzer_);
IcebergScanNode deleteScanNode = new IcebergScanNode(
deleteScanNodeId,
@@ -255,7 +258,8 @@ public class IcebergScanPlanner {
aggInfo_,
Lists.newArrayList(positionDeleteFiles_),
Collections.emptyList(), /*nonIdentityConjuncts*/
- Collections.emptyList()); /*skippedConjuncts*/
+ Collections.emptyList(), /*skippedConjuncts*/
+ snapshotId_);
deleteScanNode.init(analyzer_);
// Now let's create the JOIN node
@@ -435,7 +439,7 @@ public class IcebergScanPlanner {
PlanNodeId dataScanNodeId = ctx_.getNextNodeId();
IcebergScanNode dataScanNode = new IcebergScanNode(
dataScanNodeId, tblRef_, conjuncts_, aggInfo_, dataFilesWithDeletes_,
- nonIdentityConjuncts_, getSkippedConjuncts());
+ nonIdentityConjuncts_, getSkippedConjuncts(), snapshotId_);
addSlotsForEqualityDelete(equalityIds_, tblRef_);
dataScanNode.init(analyzer_);
@@ -464,7 +468,8 @@ public class IcebergScanPlanner {
aggInfo_,
Lists.newArrayList(equalityDeleteFiles_),
Collections.emptyList(), /*nonIdentityConjuncts*/
- Collections.emptyList()); /*skippedConjuncts*/
+ Collections.emptyList(), /*skippedConjuncts*/
+ snapshotId_);
deleteScanNode.init(analyzer_);
Pair<List<BinaryPredicate>, List<Expr>> equalityJoinConjuncts =
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index f0dba7713..ad2ec5401 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -646,6 +646,12 @@ public class IcebergUtil {
}
}
+ public static long getSnapshotId(FeIcebergTable table, TimeTravelSpec timeTravelSpec) {
+ if (timeTravelSpec == null) return table.snapshotId();
+ TableScan scan = createScanAsOf(table, timeTravelSpec);
+ return scan.snapshot().snapshotId();
+ }
+
private static TableScan createScanAsOf(FeIcebergTable table,
TimeTravelSpec timeTravelSpec) {
TableScan scan = table.getIcebergApiTable().newScan();
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
index b70eb86dd..8a3d077cc 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
@@ -965,7 +965,8 @@ public class PlannerTest extends PlannerTestBase {
TQueryOptions options = defaultQueryOptions();
runPlannerTestFile("tablesample", options,
ImmutableSet.of(PlannerTestOption.EXTENDED_EXPLAIN,
- PlannerTestOption.DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS));
+ PlannerTestOption.DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS,
+ PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS));
}
@Test
@@ -1283,7 +1284,8 @@ public class PlannerTest extends PlannerTestBase {
TQueryOptions queryOptions = new TQueryOptions();
queryOptions.setIceberg_predicate_pushdown_subsetting(false);
runPlannerTestFile("iceberg-predicates-disabled-subsetting", "functional_parquet",
- queryOptions, ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY));
+ queryOptions, ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY,
+ PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS));
}
/**
@@ -1291,8 +1293,11 @@ public class PlannerTest extends PlannerTestBase {
*/
@Test
public void testIcebergV2TableScans() {
- runPlannerTestFile("iceberg-v2-tables", "functional_parquet",
- ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY));
+ TQueryOptions options = defaultQueryOptions();
+ options.setTimezone("UTC");
+ runPlannerTestFile("iceberg-v2-tables", "functional_parquet", options,
+ ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY,
+ PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS));
}
/**
@@ -1303,7 +1308,8 @@ public class PlannerTest extends PlannerTestBase {
TQueryOptions options = defaultQueryOptions();
options.setDisable_optimized_iceberg_v2_read(true);
runPlannerTestFile("iceberg-v2-tables-hash-join", "functional_parquet", options,
- ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY));
+ ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY,
+ PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS));
}
/**
@@ -1312,7 +1318,8 @@ public class PlannerTest extends PlannerTestBase {
@Test
public void testIcebergV2Delete() {
runPlannerTestFile("iceberg-v2-delete", "functional_parquet",
- ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY));
+ ImmutableSet.of(PlannerTestOption.VALIDATE_CARDINALITY,
+ PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS));
}
@Test
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
index 53205334f..023a6147c 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
@@ -590,6 +590,9 @@ public class PlannerTestBase extends FrontendTestBase {
PlannerTestOption.DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS)) {
resultFilters.add(TestUtils.PARTITIONS_FILTER);
}
+ if (!testOptions.contains(PlannerTestOption.VALIDATE_ICEBERG_SNAPSHOT_IDS)) {
+ resultFilters.add(TestUtils.ICEBERG_SNAPSHOT_ID_FILTER);
+ }
String planDiff = TestUtils.compareOutput(
Lists.newArrayList(explainStr.split("\n")), expectedPlan, true, resultFilters);
@@ -910,7 +913,11 @@ public class PlannerTestBase extends FrontendTestBase {
DISABLE_HDFS_NUM_ROWS_ESTIMATE,
// If set, make no attempt to validate the estimated number of rows for any
// partitions in an hdfs table.
- DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS
+ DO_NOT_VALIDATE_ROWCOUNT_ESTIMATION_FOR_PARTITIONS,
+ // Verify that the snapshot ids in the plan match to the expected values. We
+ // can only do this for tests that operate on pre-written Iceberg tables,
+ // e.g. functional_parquet.iceberg_partitioned.
+ VALIDATE_ICEBERG_SNAPSHOT_IDS
}
protected void runPlannerTestFile(String testFile, TQueryOptions options) {
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index 24582f8ad..761a2ffc5 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -211,6 +211,9 @@ public class TestUtils {
public static final IgnoreValueFilter CARDINALITY_FILTER =
new IgnoreValueFilter("cardinality", "\\S+");
+ public static final IgnoreValueFilter ICEBERG_SNAPSHOT_ID_FILTER =
+ new IgnoreValueFilter("Iceberg snapshot id", " \\d+", ':');
+
// Ignore any values after 'rows=' in partitions: 0/24 rows=12.83K or
// partitions: 0/24 rows=unavailable entries
public static final ReplaceValueFilter PARTITIONS_FILTER =
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test
index 5421100c1..2bcd45ace 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates-disabled-subsetting.test
@@ -8,6 +8,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.97KB
predicates: `user` = 'Lisa', action = 'download'
+ Iceberg snapshot id: 8270633197658268308
row-size=24B cardinality=1
====
# If no residual expression remain after Iceberg's planning, all partition-based
@@ -18,6 +19,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.97KB
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action = 'download'
row-size=44B cardinality=6
====
@@ -27,6 +29,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.97KB
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action = 'download', event_time < TIMESTAMP '2022-01-01 00:00:00'
row-size=44B cardinality=1
-====
\ No newline at end of file
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test
index 8d22e0b32..9dc71186a 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test
@@ -7,6 +7,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partition_evolution]
HDFS partitions=1/1 files=730 size=1.25MB
+ Iceberg snapshot id: 188637238055891670
skipped Iceberg predicates: `year` = 2010
row-size=20B cardinality=7.30K
====
@@ -19,6 +20,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partition_evolution]
HDFS partitions=1/1 files=124 size=216.63KB
predicates: `month` = 1
+ Iceberg snapshot id: 188637238055891670
row-size=24B cardinality=1.24K
====
# The predicates that couldn't be pushed to Iceberg are pushed down to the scan node,
@@ -31,6 +33,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partition_evolution]
HDFS partitions=1/1 files=730 size=1.25MB
predicates: power(id, 3) > 1000
+ Iceberg snapshot id: 188637238055891670
skipped Iceberg predicates: `year` = 2010
row-size=20B cardinality=730
====
@@ -43,6 +46,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partition_evolution]
HDFS partitions=1/1 files=730 size=1.25MB
predicates: id > 1000
+ Iceberg snapshot id: 188637238055891670
skipped Iceberg predicates: `year` = 2010
row-size=20B cardinality=730
====
@@ -57,6 +61,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partition_evolution]
HDFS partitions=1/1 files=2 size=3.49KB
predicates: date_string_col = '061610'
+ Iceberg snapshot id: 188637238055891670
skipped Iceberg predicates: `year` = 2010
row-size=40B cardinality=2
====
@@ -72,6 +77,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.85KB
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action = 'click'
row-size=8B cardinality=6
====
@@ -85,6 +91,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.85KB
predicates: `user` = NULL
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action = 'click'
row-size=12B cardinality=1
---- DISTRIBUTEDPLAN
@@ -95,6 +102,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.85KB
predicates: `user` = NULL
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action = 'click'
row-size=12B cardinality=1
====
@@ -106,6 +114,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=20 size=22.90KB
predicates: action = NULL
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: event_time < TIMESTAMP '2022-01-01 00:00:00'
row-size=44B cardinality=2
====
@@ -118,6 +127,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=9 size=10.33KB
predicates: id < 10, action LIKE 'd%'
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: event_time < TIMESTAMP '2022-01-01 00:00:00'
row-size=44B cardinality=1
====
@@ -129,6 +139,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=10 size=11.42KB
predicates: (event_time < TIMESTAMP '2020-01-01 00:00:00' OR id > 10), action LIKE 'd%'
+ Iceberg snapshot id: 8270633197658268308
row-size=44B cardinality=1
====
# Predicate on a partition introduced by partition evolution pushed down to the scan node and
@@ -140,6 +151,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partition_evolution]
HDFS partitions=1/1 files=62 size=108.30KB
predicates: `month` = 12
+ Iceberg snapshot id: 188637238055891670
skipped Iceberg predicates: `year` = 2010
row-size=40B cardinality=620
====
@@ -153,6 +165,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=12 size=13.65KB
predicates: action IN ('click', 'view'), (id > 5 OR (id < 2))
+ Iceberg snapshot id: 8270633197658268308
row-size=44B cardinality=1
====
# IS NOT NULL predicate skipped on partitioned column 'action', predicate on
@@ -164,10 +177,10 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=9 size=10.33KB
predicates: id < 10
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IS NOT NULL
row-size=44B cardinality=1
====
----- QUERY
# Regression test for IMPALA-12580: predicate filtering could throw UnsupportedOperationException
select p_int from functional_parquet.iceberg_alltypes_part where i = 1 and p_int = i;
---- PLAN
@@ -176,10 +189,10 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_alltypes_part]
HDFS partitions=1/1 files=1 size=433B
predicates: i = 1
+ Iceberg snapshot id: 6167994413873848621
skipped Iceberg predicates: functional_parquet.iceberg_alltypes_part.p_int = 1
row-size=8B cardinality=1
====
----- QUERY
# Regression test for IMPALA-12580: predicate filtering could throw UnsupportedOperationException
select * from functional_parquet.iceberg_alltypes_part
where i = 1 and p_bigint = 10 + i and p_int = p_bigint - 10;
@@ -189,10 +202,10 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_alltypes_part]
HDFS partitions=1/1 files=1 size=433B
predicates: i = 1
+ Iceberg snapshot id: 6167994413873848621
skipped Iceberg predicates: p_bigint = 11, p_int = 1
row-size=49B cardinality=1
====
----- QUERY
select * from functional_parquet.iceberg_alltypes_part
where p_string = concat(chr(cast(p_int + 104 as int)), "mpala") and
p_string = concat("i", "mpala") and p_decimal between 122 and 124 and i = floor(p_float)
@@ -202,10 +215,10 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_alltypes_part]
HDFS partitions=1/1 files=1 size=433B
predicates: i = floor(p_float), concat(chr(CAST(p_int + 104 AS INT)), 'mpala') = 'impala'
+ Iceberg snapshot id: 6167994413873848621
skipped Iceberg predicates: p_string = 'impala', p_decimal >= 122, p_decimal <= 124
row-size=49B cardinality=1
====
----- QUERY
select * from functional_parquet.iceberg_alltypes_part where i in (1,2,3) and i = p_int;
---- PLAN
PLAN-ROOT SINK
@@ -213,6 +226,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_alltypes_part]
HDFS partitions=1/1 files=1 size=433B
predicates: i = p_int, i IN (1, 2, 3)
+ Iceberg snapshot id: 6167994413873848621
skipped Iceberg predicates: functional_parquet.iceberg_alltypes_part.p_int IN (1, 2, 3)
row-size=49B cardinality=1
-====
\ No newline at end of file
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test
index 6eac5b341..c75cdaea0 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-delete.test
@@ -9,6 +9,7 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_no_deletes-POSITION-DELETE]
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i = 3
+ Iceberg snapshot id: 728158873687794725
row-size=24B cardinality=1
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_no_deletes-POSITION-DELETE]
@@ -20,6 +21,7 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_no_deletes-POSITION-DELETE]
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i = 3
+ Iceberg snapshot id: 728158873687794725
row-size=24B cardinality=1
====
DELETE FROM iceberg_v2_delete_positional where id = 15
@@ -35,11 +37,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: id = 15
+ Iceberg snapshot id: 5725822353600261755
row-size=28B cardinality=1
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE]
@@ -55,11 +59,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: id = 15
+ Iceberg snapshot id: 5725822353600261755
row-size=28B cardinality=1
====
DELETE FROM iceberg_v2_delete_positional
@@ -85,10 +91,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
| |
| |--04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete]
| | HDFS partitions=1/1 files=1 size=1.54KB
+| | Iceberg snapshot id: 5725822353600261755
| | row-size=182B cardinality=1
| |
| 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
| HDFS partitions=1/1 files=1 size=662B
+| Iceberg snapshot id: 5725822353600261755
| row-size=28B cardinality=3
|
02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN]
@@ -96,11 +104,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
runtime filters: RF000 -> id
+ Iceberg snapshot id: 5725822353600261755
row-size=28B cardinality=3
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE]
@@ -133,10 +143,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
| | |
| | 04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete]
| | HDFS partitions=1/1 files=1 size=1.54KB
+| | Iceberg snapshot id: 5725822353600261755
| | row-size=182B cardinality=1
| |
| 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
| HDFS partitions=1/1 files=1 size=662B
+| Iceberg snapshot id: 5725822353600261755
| row-size=28B cardinality=3
|
02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED]
@@ -146,11 +158,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
runtime filters: RF000 -> id
+ Iceberg snapshot id: 5725822353600261755
row-size=28B cardinality=3
====
DELETE FROM iceberg_v2_delete_positional WHERE FILE__POSITION = id
@@ -166,11 +180,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: FILE__POSITION = id
+ Iceberg snapshot id: 5725822353600261755
row-size=28B cardinality=1
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE]
@@ -186,11 +202,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_delete_positional-POSITION-DE
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: FILE__POSITION = id
+ Iceberg snapshot id: 5725822353600261755
row-size=28B cardinality=1
====
delete from iceberg_v2_partitioned_position_deletes where id = 20;
@@ -206,11 +224,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
predicates: id = 20
+ Iceberg snapshot id: 8885697082976537578
row-size=40B cardinality=2
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE]
@@ -228,11 +248,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
predicates: id = 20
+ Iceberg snapshot id: 8885697082976537578
row-size=40B cardinality=2
====
delete from iceberg_v2_partitioned_position_deletes where action = 'click';
@@ -248,10 +270,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.15KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.15KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'click'
row-size=36B cardinality=6
---- DISTRIBUTEDPLAN
@@ -268,10 +292,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.15KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.15KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'click'
row-size=36B cardinality=6
====
@@ -288,11 +314,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
predicates: `user` LIKE 'A%'
+ Iceberg snapshot id: 8885697082976537578
row-size=48B cardinality=2
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE]
@@ -310,11 +338,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
predicates: `user` LIKE 'A%'
+ Iceberg snapshot id: 8885697082976537578
row-size=48B cardinality=2
====
delete from iceberg_v2_partitioned_position_deletes
@@ -340,10 +370,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
| |
| |--04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete]
| | HDFS partitions=1/1 files=1 size=1.54KB
+| | Iceberg snapshot id: 5725822353600261755
| | row-size=182B cardinality=1
| |
| 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
| HDFS partitions=1/1 files=1 size=662B
+| Iceberg snapshot id: 5725822353600261755
| row-size=28B cardinality=3
|
02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN]
@@ -351,11 +383,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
runtime filters: RF000 -> id
+ Iceberg snapshot id: 8885697082976537578
row-size=40B cardinality=20
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE]
@@ -390,10 +424,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
| | |
| | 04:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-04 functional_parquet.iceberg_v2_delete_positional-position-delete]
| | HDFS partitions=1/1 files=1 size=1.54KB
+| | Iceberg snapshot id: 5725822353600261755
| | row-size=182B cardinality=1
| |
| 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
| HDFS partitions=1/1 files=1 size=662B
+| Iceberg snapshot id: 5725822353600261755
| row-size=28B cardinality=3
|
02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED]
@@ -403,11 +439,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
runtime filters: RF000 -> id
+ Iceberg snapshot id: 8885697082976537578
row-size=40B cardinality=20
====
DELETE FROM iceberg_v2_partitioned_position_deletes WHERE FILE__POSITION = id
@@ -423,11 +461,13 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
predicates: FILE__POSITION = id
+ Iceberg snapshot id: 8885697082976537578
row-size=40B cardinality=2
---- DISTRIBUTEDPLAN
DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE]
@@ -445,10 +485,12 @@ DELETE FROM ICEBERG [functional_parquet.iceberg_v2_partitioned_position_deletes-
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
predicates: FILE__POSITION = id
+ Iceberg snapshot id: 8885697082976537578
row-size=40B cardinality=2
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test
index 51b7b8c46..a43b98c5b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables-hash-join.test
@@ -23,6 +23,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i > 1
+ Iceberg snapshot id: 728158873687794725
row-size=4B cardinality=1
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -40,6 +41,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i > 1
+ Iceberg snapshot id: 728158873687794725
row-size=4B cardinality=1
====
SELECT count(*) from iceberg_v2_delete_positional;
@@ -55,10 +57,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=20B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -80,10 +84,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=20B cardinality=3
====
SELECT * from iceberg_v2_delete_positional;
@@ -95,10 +101,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -112,10 +120,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=3
====
SELECT * from iceberg_v2_positional_delete_all_rows;
@@ -127,10 +137,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -144,10 +156,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
====
SELECT * from iceberg_v2_no_deletes limit 1
@@ -156,6 +170,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 728158873687794725
limit: 1
row-size=16B cardinality=1
---- DISTRIBUTEDPLAN
@@ -166,6 +181,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 728158873687794725
limit: 1
row-size=16B cardinality=1
====
@@ -179,10 +195,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -198,10 +216,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
====
SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files limit 1
@@ -218,14 +238,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -245,16 +268,19 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files
@@ -270,14 +296,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -295,16 +324,19 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 7490459762454857930;
@@ -334,10 +366,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=1 size=2.63KB
+| Iceberg snapshot id: 752781918366351945
| row-size=267B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 752781918366351945
row-size=20B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -359,10 +393,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=1 size=2.63KB
+| Iceberg snapshot id: 752781918366351945
| row-size=267B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 752781918366351945
row-size=20B cardinality=3
====
SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files;
@@ -378,10 +414,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -403,12 +441,14 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
04:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
====
SELECT * from iceberg_v2_positional_update_all_rows
@@ -424,14 +464,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | HDFS partitions=1/1 files=1 size=2.60KB
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=246B cardinality=3
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=36B cardinality=3
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 3877007445826010687
row-size=36B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -449,14 +492,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | HDFS partitions=1/1 files=1 size=2.60KB
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=246B cardinality=3
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=36B cardinality=3
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 3877007445826010687
row-size=36B cardinality=3
====
SELECT * from iceberg_v2_partitioned_position_deletes
@@ -468,10 +514,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
+ Iceberg snapshot id: 8885697082976537578
row-size=64B cardinality=20
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -485,12 +533,14 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
03:EXCHANGE [HASH(functional_parquet.iceberg_v2_partitioned_position_deletes.file__position,functional_parquet.iceberg_v2_partitioned_position_deletes.input__file__name)]
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
+ Iceberg snapshot id: 8885697082976537578
row-size=64B cardinality=20
====
SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files
@@ -507,16 +557,19 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i > 2
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i > 2
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -534,6 +587,7 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
@@ -541,11 +595,13 @@ PLAN-ROOT SINK
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i > 2
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i > 2
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
====
select * from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 1497619269847778439
@@ -560,6 +616,7 @@ PLAN-ROOT SINK
|
|--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 4363979609026842966
| row-size=16B cardinality=3
|
05:AGGREGATE [FINALIZE]
@@ -575,14 +632,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -597,6 +657,7 @@ PLAN-ROOT SINK
| |
| 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 4363979609026842966
| row-size=16B cardinality=3
|
11:AGGREGATE [FINALIZE]
@@ -620,16 +681,19 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 08:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
with v as (select i + 1000 as ii, upper(s) as ss from iceberg_v2_positional_not_all_data_files_have_delete_files)
@@ -646,16 +710,19 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i + 1000 > 1003
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i + 1000 > 1003
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -673,6 +740,7 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
@@ -680,11 +748,13 @@ PLAN-ROOT SINK
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i + 1000 > 1003
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i + 1000 > 1003
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
====
select *
@@ -711,14 +781,17 @@ PLAN-ROOT SINK
| | |
| | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | | HDFS partitions=1/1 files=1 size=2.60KB
+| | | Iceberg snapshot id: 3877007445826010687
| | | row-size=246B cardinality=3
| | |
| | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| | HDFS partitions=1/1 files=1 size=625B
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=24B cardinality=3
| |
| 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=24B cardinality=3
|
04:UNION
@@ -730,16 +803,19 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| runtime filters: RF000 -> i
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
runtime filters: RF000 -> i
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -774,14 +850,17 @@ PLAN-ROOT SINK
| | | |
| | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | | HDFS partitions=1/1 files=1 size=2.60KB
+| | | Iceberg snapshot id: 3877007445826010687
| | | row-size=246B cardinality=3
| | |
| | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| | HDFS partitions=1/1 files=1 size=625B
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=24B cardinality=3
| |
| 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=24B cardinality=3
|
04:UNION
@@ -795,6 +874,7 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 12:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
@@ -802,11 +882,13 @@ PLAN-ROOT SINK
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| runtime filters: RF000 -> i
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
runtime filters: RF000 -> i
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
select * from iceberg_v2_partitioned_position_deletes where action = 'download' and id > 0;
@@ -818,11 +900,13 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: id > 0
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
---- DISTRIBUTEDPLAN
@@ -837,11 +921,13 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: id > 0
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
====
@@ -855,11 +941,13 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: `user` = 'Lisa'
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
---- DISTRIBUTEDPLAN
@@ -874,11 +962,13 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: `user` = 'Lisa'
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
====
@@ -889,6 +979,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -899,6 +990,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
====
@@ -909,6 +1001,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -919,6 +1012,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
====
@@ -929,6 +1023,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.85KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -939,6 +1034,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.85KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click')
row-size=32B cardinality=1
====
@@ -949,6 +1045,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -959,6 +1056,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view')
row-size=32B cardinality=1
====
@@ -969,6 +1067,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view'))
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -979,6 +1078,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view'))
row-size=32B cardinality=1
====
@@ -989,6 +1089,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=20 size=22.90KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a')
row-size=32B cardinality=2
---- DISTRIBUTEDPLAN
@@ -999,6 +1100,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=20 size=22.90KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a')
row-size=32B cardinality=2
====
@@ -1009,6 +1111,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=0 size=0B
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00'
row-size=28B cardinality=0
---- DISTRIBUTEDPLAN
@@ -1018,6 +1121,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=0 size=0B
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00'
row-size=28B cardinality=0
====
@@ -1031,10 +1135,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=6
---- DISTRIBUTEDPLAN
@@ -1049,10 +1155,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=6
====
@@ -1074,10 +1182,12 @@ PLAN-ROOT SINK
| |
| |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=20B cardinality=6
|
09:NESTED LOOP JOIN [CROSS JOIN]
@@ -1096,10 +1206,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -1126,12 +1238,14 @@ PLAN-ROOT SINK
| | |
| | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 16:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
| |
| 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=20B cardinality=6
|
09:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
@@ -1160,11 +1274,13 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
11:EXCHANGE [HASH(functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position,functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name)]
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test
index 29578ffe2..22da72d9b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test
@@ -23,6 +23,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i > 1
+ Iceberg snapshot id: 728158873687794725
row-size=4B cardinality=1
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -40,6 +41,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i > 1
+ Iceberg snapshot id: 728158873687794725
row-size=4B cardinality=1
====
SELECT count(*) from iceberg_v2_delete_positional;
@@ -55,10 +57,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=20B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -80,10 +84,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=20B cardinality=3
====
SELECT * from iceberg_v2_delete_positional;
@@ -95,10 +101,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -112,10 +120,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=3
====
SELECT * from iceberg_v2_positional_delete_all_rows;
@@ -127,10 +137,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -144,10 +156,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
====
SELECT * from iceberg_v2_no_deletes limit 1
@@ -156,6 +170,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 728158873687794725
limit: 1
row-size=16B cardinality=1
---- DISTRIBUTEDPLAN
@@ -166,6 +181,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 728158873687794725
limit: 1
row-size=16B cardinality=1
====
@@ -179,10 +195,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -198,10 +216,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_delete_all_rows-position-delete]
| HDFS partitions=1/1 files=1 size=2.60KB
+| Iceberg snapshot id: 444149380144800647
| row-size=246B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_delete_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 444149380144800647
row-size=36B cardinality=3
====
SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files limit 1
@@ -218,14 +238,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -245,14 +268,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files
@@ -268,14 +294,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -293,14 +322,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 7490459762454857930;
@@ -330,10 +362,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=1 size=2.63KB
+| Iceberg snapshot id: 752781918366351945
| row-size=267B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 752781918366351945
row-size=20B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -355,10 +389,60 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=1 size=2.63KB
+| Iceberg snapshot id: 752781918366351945
| row-size=267B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 752781918366351945
+ row-size=20B cardinality=3
+====
+SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files for system_time as of "2022-08-19 13:50:00";
+---- PLAN
+PLAN-ROOT SINK
+|
+03:AGGREGATE [FINALIZE]
+| output: count(*)
+| row-size=8B cardinality=1
+|
+02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN]
+| row-size=20B cardinality=2
+|
+|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
+| HDFS partitions=1/1 files=1 size=2.63KB
+| Iceberg snapshot id: 752781918366351945
+| row-size=267B cardinality=1
+|
+00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
+ HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 752781918366351945
+ row-size=20B cardinality=3
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+06:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| row-size=8B cardinality=1
+|
+05:EXCHANGE [UNPARTITIONED]
+|
+03:AGGREGATE
+| output: count(*)
+| row-size=8B cardinality=1
+|
+02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED]
+| row-size=20B cardinality=2
+|
+|--04:EXCHANGE [DIRECTED]
+| |
+| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
+| HDFS partitions=1/1 files=1 size=2.63KB
+| Iceberg snapshot id: 752781918366351945
+| row-size=267B cardinality=1
+|
+00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
+ HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 752781918366351945
row-size=20B cardinality=3
====
SELECT count(*) from iceberg_v2_positional_not_all_data_files_have_delete_files;
@@ -374,10 +458,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -399,10 +485,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
====
SELECT * from iceberg_v2_positional_update_all_rows
@@ -418,14 +506,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | HDFS partitions=1/1 files=1 size=2.60KB
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=246B cardinality=3
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=36B cardinality=3
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 3877007445826010687
row-size=36B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -443,14 +534,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | HDFS partitions=1/1 files=1 size=2.60KB
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=246B cardinality=3
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=36B cardinality=3
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
HDFS partitions=1/1 files=1 size=625B
+ Iceberg snapshot id: 3877007445826010687
row-size=36B cardinality=3
====
SELECT * from iceberg_v2_partitioned_position_deletes
@@ -462,10 +556,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
+ Iceberg snapshot id: 8885697082976537578
row-size=64B cardinality=20
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -479,10 +575,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
+ Iceberg snapshot id: 8885697082976537578
row-size=64B cardinality=20
====
SELECT * from iceberg_v2_positional_not_all_data_files_have_delete_files
@@ -499,16 +597,19 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i > 2
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i > 2
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -526,16 +627,19 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i > 2
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i > 2
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
====
select * from iceberg_v2_positional_not_all_data_files_have_delete_files for system_version as of 1497619269847778439
@@ -550,6 +654,7 @@ PLAN-ROOT SINK
|
|--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 4363979609026842966
| row-size=16B cardinality=3
|
05:AGGREGATE [FINALIZE]
@@ -565,14 +670,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -587,6 +695,7 @@ PLAN-ROOT SINK
| |
| 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 4363979609026842966
| row-size=16B cardinality=3
|
10:AGGREGATE [FINALIZE]
@@ -610,14 +719,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
with v as (select i + 1000 as ii, upper(s) as ss from iceberg_v2_positional_not_all_data_files_have_delete_files)
@@ -634,16 +746,19 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i + 1000 > 1003
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i + 1000 > 1003
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -661,16 +776,19 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| predicates: i + 1000 > 1003
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
predicates: i + 1000 > 1003
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=1
====
select *
@@ -697,14 +815,17 @@ PLAN-ROOT SINK
| | |
| | |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | | HDFS partitions=1/1 files=1 size=2.60KB
+| | | Iceberg snapshot id: 3877007445826010687
| | | row-size=246B cardinality=3
| | |
| | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| | HDFS partitions=1/1 files=1 size=625B
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=24B cardinality=3
| |
| 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=24B cardinality=3
|
04:UNION
@@ -716,16 +837,19 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| runtime filters: RF000 -> i
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
runtime filters: RF000 -> i
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -760,14 +884,17 @@ PLAN-ROOT SINK
| | | |
| | | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_update_all_rows-position-delete]
| | | HDFS partitions=1/1 files=1 size=2.60KB
+| | | Iceberg snapshot id: 3877007445826010687
| | | row-size=246B cardinality=3
| | |
| | 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| | HDFS partitions=1/1 files=1 size=625B
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=24B cardinality=3
| |
| 08:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=24B cardinality=3
|
04:UNION
@@ -781,16 +908,19 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
| runtime filters: RF000 -> i
+| Iceberg snapshot id: 1497619269847778439
| row-size=36B cardinality=6
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
runtime filters: RF000 -> i
+ Iceberg snapshot id: 1497619269847778439
row-size=36B cardinality=4
====
select * from iceberg_v2_partitioned_position_deletes where action = 'download' and id > 0;
@@ -802,11 +932,13 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: id > 0
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
---- DISTRIBUTEDPLAN
@@ -821,11 +953,13 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: id > 0
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
====
@@ -839,11 +973,13 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: `user` = 'Lisa'
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
---- DISTRIBUTEDPLAN
@@ -858,11 +994,13 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
predicates: `user` = 'Lisa'
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=1
====
@@ -873,6 +1011,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -883,6 +1022,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
====
@@ -893,6 +1033,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -903,6 +1044,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action IN ('click', 'view')
row-size=32B cardinality=1
====
@@ -913,6 +1055,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.85KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -923,6 +1066,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=6 size=6.85KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click')
row-size=32B cardinality=1
====
@@ -933,6 +1077,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view')
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -943,6 +1088,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action = 'click' OR action = 'view')
row-size=32B cardinality=1
====
@@ -953,6 +1099,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view'))
row-size=32B cardinality=1
---- DISTRIBUTEDPLAN
@@ -963,6 +1110,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=14 size=15.93KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action IN ('click', 'view'))
row-size=32B cardinality=1
====
@@ -973,6 +1121,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=20 size=22.90KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a')
row-size=32B cardinality=2
---- DISTRIBUTEDPLAN
@@ -983,6 +1132,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=20 size=22.90KB
predicates: id > 0
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: (event_time = TIMESTAMP '2020-01-01 11:00:00' OR action > 'a')
row-size=32B cardinality=2
====
@@ -993,6 +1143,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=0 size=0B
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00'
row-size=28B cardinality=0
---- DISTRIBUTEDPLAN
@@ -1002,6 +1153,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=0 size=0B
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: event_time = TIMESTAMP '2020-01-01 11:00:00'
row-size=28B cardinality=0
====
@@ -1015,10 +1167,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=6
---- DISTRIBUTEDPLAN
@@ -1033,10 +1187,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.18KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=207B cardinality=2
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.17KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'download'
row-size=64B cardinality=6
====
@@ -1058,10 +1214,12 @@ PLAN-ROOT SINK
| |
| |--06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=20B cardinality=6
|
09:NESTED LOOP JOIN [CROSS JOIN]
@@ -1080,10 +1238,12 @@ PLAN-ROOT SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -1110,10 +1270,12 @@ PLAN-ROOT SINK
| | |
| | 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-06 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | row-size=267B cardinality=4
| |
| 05:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=2 size=1.22KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=20B cardinality=6
|
09:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
@@ -1142,10 +1304,12 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| HDFS partitions=1/1 files=2 size=5.33KB
+| Iceberg snapshot id: 1497619269847778439
| row-size=267B cardinality=4
|
00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=2 size=1.22KB
+ Iceberg snapshot id: 1497619269847778439
row-size=20B cardinality=6
====
select * from functional_parquet.iceberg_v2_delete_equality;
@@ -1157,20 +1321,23 @@ PLAN-ROOT SINK
| row-size=28B cardinality=3
|
|--02:HASH JOIN [LEFT ANTI JOIN]
-| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id
+| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id
| | other join predicates: functional_parquet.iceberg_v2_delete_equality.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality-equality-delete-01.iceberg__data__sequence__number
| | row-size=28B cardinality=1
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
| | HDFS partitions=1/1 files=1 size=457B
+| | Iceberg snapshot id: 5763349507283783091
| | row-size=16B cardinality=1
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
| HDFS partitions=1/1 files=1 size=709B
+| Iceberg snapshot id: 5763349507283783091
| row-size=28B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
HDFS partitions=1/1 files=2 size=1.41KB
+ Iceberg snapshot id: 5763349507283783091
row-size=28B cardinality=2
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -1190,14 +1357,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
| | HDFS partitions=1/1 files=1 size=457B
+| | Iceberg snapshot id: 5763349507283783091
| | row-size=16B cardinality=1
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
| HDFS partitions=1/1 files=1 size=709B
+| Iceberg snapshot id: 5763349507283783091
| row-size=28B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
HDFS partitions=1/1 files=2 size=1.41KB
+ Iceberg snapshot id: 5763349507283783091
row-size=28B cardinality=2
====
select * from functional_parquet.iceberg_v2_delete_equality for system_version as of 5763349507283783091;
@@ -1215,14 +1385,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
| | HDFS partitions=1/1 files=1 size=457B
+| | Iceberg snapshot id: 5763349507283783091
| | row-size=16B cardinality=1
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
| HDFS partitions=1/1 files=1 size=709B
+| Iceberg snapshot id: 5763349507283783091
| row-size=28B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
HDFS partitions=1/1 files=2 size=1.41KB
+ Iceberg snapshot id: 5763349507283783091
row-size=28B cardinality=2
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -1242,14 +1415,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
| | HDFS partitions=1/1 files=1 size=457B
+| | Iceberg snapshot id: 5763349507283783091
| | row-size=16B cardinality=1
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
| HDFS partitions=1/1 files=1 size=709B
+| Iceberg snapshot id: 5763349507283783091
| row-size=28B cardinality=1
|
03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
HDFS partitions=1/1 files=2 size=1.41KB
+ Iceberg snapshot id: 5763349507283783091
row-size=28B cardinality=2
====
# Table has both equality and positional delete files. Equality fields are (i int, d date).
@@ -1268,6 +1444,7 @@ PLAN-ROOT SINK
| |
| |--03:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-EQUALITY-DELETE-03 functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03]
| | HDFS partitions=1/1 files=1 size=657B
+| | Iceberg snapshot id: 911559291487642581
| | row-size=16B cardinality=2
| |
| 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN]
@@ -1275,14 +1452,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_both_eq_and_pos-position-delete]
| | HDFS partitions=1/1 files=1 size=1.57KB
+| | Iceberg snapshot id: 911559291487642581
| | row-size=187B cardinality=1
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
| HDFS partitions=1/1 files=1 size=898B
+| Iceberg snapshot id: 911559291487642581
| row-size=48B cardinality=2
|
05:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
HDFS partitions=1/1 files=1 size=885B
+ Iceberg snapshot id: 911559291487642581
row-size=48B cardinality=2
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -1302,6 +1482,7 @@ PLAN-ROOT SINK
| | |
| | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-EQUALITY-DELETE-03 functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03]
| | HDFS partitions=1/1 files=1 size=657B
+| | Iceberg snapshot id: 911559291487642581
| | row-size=16B cardinality=2
| |
| 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED]
@@ -1311,14 +1492,17 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_both_eq_and_pos-position-delete]
| | HDFS partitions=1/1 files=1 size=1.57KB
+| | Iceberg snapshot id: 911559291487642581
| | row-size=187B cardinality=1
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
| HDFS partitions=1/1 files=1 size=898B
+| Iceberg snapshot id: 911559291487642581
| row-size=48B cardinality=2
|
05:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
HDFS partitions=1/1 files=1 size=885B
+ Iceberg snapshot id: 911559291487642581
row-size=48B cardinality=2
====
# Partitioned table with equality deletes.
@@ -1337,14 +1521,17 @@ PLAN-ROOT SINK
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01]
| | HDFS partitions=1/1 files=2 size=1.33KB
+| | Iceberg snapshot id: 4821964189199835313
| | row-size=24B cardinality=3
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
| HDFS partitions=1/1 files=2 size=1.76KB
+| Iceberg snapshot id: 4821964189199835313
| row-size=28B cardinality=5
|
03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
HDFS partitions=1/1 files=2 size=1.71KB
+ Iceberg snapshot id: 4821964189199835313
row-size=28B cardinality=3
---- DISTRIBUTEDPLAN
PLAN-ROOT SINK
@@ -1364,15 +1551,18 @@ PLAN-ROOT SINK
| | |
| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01]
| | HDFS partitions=1/1 files=2 size=1.33KB
+| | Iceberg snapshot id: 4821964189199835313
| | row-size=24B cardinality=3
| |
| 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_delete_equality_partitioned.d,functional_parquet.iceberg_v2_delete_equality_partitioned.s)]
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
| HDFS partitions=1/1 files=2 size=1.76KB
+| Iceberg snapshot id: 4821964189199835313
| row-size=28B cardinality=5
|
03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
HDFS partitions=1/1 files=2 size=1.71KB
+ Iceberg snapshot id: 4821964189199835313
row-size=28B cardinality=3
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test
index a7b61a2a6..57c27f3ef 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-update.test
@@ -7,6 +7,7 @@ MULTI DATA SINK
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i = 3
+ Iceberg snapshot id: 728158873687794725
row-size=36B cardinality=1
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -16,6 +17,7 @@ MULTI DATA SINK
00:SCAN HDFS [functional_parquet.iceberg_v2_no_deletes]
HDFS partitions=1/1 files=1 size=625B
predicates: i = 3
+ Iceberg snapshot id: 728158873687794725
row-size=36B cardinality=1
====
UPDATE iceberg_v2_delete_positional SET `data` = concat(`data`,'a') where id = 15
@@ -29,11 +31,13 @@ MULTI DATA SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: id = 15
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=1
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -47,11 +51,13 @@ MULTI DATA SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: id = 15
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=1
====
UPDATE iceberg_v2_delete_positional SET id = cast(id+1 as int)
@@ -65,10 +71,12 @@ MULTI DATA SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=3
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -82,10 +90,12 @@ MULTI DATA SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=3
====
UPDATE iceberg_v2_delete_positional SET id = 42 WHERE FILE__POSITION = id
@@ -99,11 +109,13 @@ MULTI DATA SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: FILE__POSITION = id
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=1
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -117,11 +129,13 @@ MULTI DATA SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_positional-position-delete]
| HDFS partitions=1/1 files=1 size=1.54KB
+| Iceberg snapshot id: 5725822353600261755
| row-size=182B cardinality=1
|
00:SCAN HDFS [functional_parquet.iceberg_v2_delete_positional]
HDFS partitions=1/1 files=1 size=662B
predicates: FILE__POSITION = id
+ Iceberg snapshot id: 5725822353600261755
row-size=40B cardinality=1
====
UPDATE iceberg_v2_partitioned_position_deletes set id = length(action)
@@ -139,10 +153,12 @@ MULTI DATA SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
+ Iceberg snapshot id: 8885697082976537578
row-size=76B cardinality=20
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -162,10 +178,12 @@ MULTI DATA SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=3 size=3.48KB
+ Iceberg snapshot id: 8885697082976537578
row-size=76B cardinality=20
====
UPDATE iceberg_v2_partitioned_position_deletes set id = length(action) where action = 'click'
@@ -183,10 +201,12 @@ MULTI DATA SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.15KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.15KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'click'
row-size=76B cardinality=6
---- DISTRIBUTEDPLAN
@@ -205,10 +225,12 @@ MULTI DATA SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 functional_parquet.iceberg_v2_partitioned_position_deletes-position-delete]
| HDFS partitions=1/1 files=1 size=3.15KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=3
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes]
HDFS partitions=1/1 files=1 size=1.15KB
+ Iceberg snapshot id: 8885697082976537578
skipped Iceberg predicates: action = 'click'
row-size=76B cardinality=6
====
@@ -236,14 +258,17 @@ MULTI DATA SINK
| | |
| | |--04:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-04 source-position-delete]
| | | HDFS partitions=1/1 files=1 size=2.60KB
+| | | Iceberg snapshot id: 3877007445826010687
| | | row-size=246B cardinality=3
| | |
| | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source]
| | HDFS partitions=1/1 files=1 size=625B
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=36B cardinality=3
| |
| 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=36B cardinality=3
|
02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN]
@@ -251,11 +276,13 @@ MULTI DATA SINK
|
|--01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 target-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes target]
HDFS partitions=1/1 files=3 size=3.48KB
runtime filters: RF000 -> target.id
+ Iceberg snapshot id: 8885697082976537578
row-size=68B cardinality=20
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -286,14 +313,17 @@ MULTI DATA SINK
| | | |
| | | 04:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows-POSITION-DELETE-04 source-position-delete]
| | | HDFS partitions=1/1 files=1 size=2.60KB
+| | | Iceberg snapshot id: 3877007445826010687
| | | row-size=246B cardinality=3
| | |
| | 03:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source]
| | HDFS partitions=1/1 files=1 size=625B
+| | Iceberg snapshot id: 3877007445826010687
| | row-size=36B cardinality=3
| |
| 06:SCAN HDFS [functional_parquet.iceberg_v2_positional_update_all_rows source]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 3877007445826010687
| row-size=36B cardinality=3
|
02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED]
@@ -303,11 +333,13 @@ MULTI DATA SINK
| |
| 01:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes-POSITION-DELETE-01 target-position-delete]
| HDFS partitions=1/1 files=3 size=9.47KB
+| Iceberg snapshot id: 8885697082976537578
| row-size=204B cardinality=10
|
00:SCAN HDFS [functional_parquet.iceberg_v2_partitioned_position_deletes target]
HDFS partitions=1/1 files=3 size=3.48KB
runtime filters: RF000 -> target.id
+ Iceberg snapshot id: 8885697082976537578
row-size=68B cardinality=20
====
update iceberg_partition_transforms_zorder set ts = days_add(ts, 10), i = cast(i + 1000 as int)
@@ -322,6 +354,7 @@ MULTI DATA SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder]
HDFS partitions=1/1 files=1 size=1.08KB
+ Iceberg snapshot id: 7350750578864730166
row-size=72B cardinality=1
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -334,6 +367,7 @@ MULTI DATA SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder]
HDFS partitions=1/1 files=1 size=1.08KB
+ Iceberg snapshot id: 7350750578864730166
row-size=72B cardinality=1
====
update ice_zorder set j = length(action)
@@ -355,11 +389,13 @@ MULTI DATA SINK
|
|--00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder]
| HDFS partitions=1/1 files=1 size=1.08KB
+| Iceberg snapshot id: 7350750578864730166
| row-size=68B cardinality=1
|
01:SCAN HDFS [functional_parquet.iceberg_partitioned source]
HDFS partitions=1/1 files=20 size=22.90KB
runtime filters: RF000 -> source.id
+ Iceberg snapshot id: 8270633197658268308
row-size=16B cardinality=20
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -381,11 +417,13 @@ MULTI DATA SINK
| |
| 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder]
| HDFS partitions=1/1 files=1 size=1.08KB
+| Iceberg snapshot id: 7350750578864730166
| row-size=68B cardinality=1
|
01:SCAN HDFS [functional_parquet.iceberg_partitioned source]
HDFS partitions=1/1 files=20 size=22.90KB
runtime filters: RF000 -> source.id
+ Iceberg snapshot id: 8270633197658268308
row-size=16B cardinality=20
====
update ice_zorder set j = length(action)
@@ -407,11 +445,13 @@ MULTI DATA SINK
|
|--00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder]
| HDFS partitions=1/1 files=1 size=1.08KB
+| Iceberg snapshot id: 7350750578864730166
| row-size=68B cardinality=1
|
01:SCAN HDFS [functional_parquet.iceberg_partitioned source]
HDFS partitions=1/1 files=20 size=22.90KB
runtime filters: RF000 -> source.id
+ Iceberg snapshot id: 8270633197658268308
row-size=16B cardinality=20
---- DISTRIBUTEDPLAN
MULTI DATA SINK
@@ -433,10 +473,12 @@ MULTI DATA SINK
| |
| 00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder ice_zorder]
| HDFS partitions=1/1 files=1 size=1.08KB
+| Iceberg snapshot id: 7350750578864730166
| row-size=68B cardinality=1
|
01:SCAN HDFS [functional_parquet.iceberg_partitioned source]
HDFS partitions=1/1 files=20 size=22.90KB
runtime filters: RF000 -> source.id
+ Iceberg snapshot id: 8270633197658268308
row-size=16B cardinality=20
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test b/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test
index 45c5d186e..7aa8965e3 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/insert-sort-by-zorder.test
@@ -401,6 +401,7 @@ WRITE TO HDFS [functional_parquet.iceberg_partition_transforms_zorder, OVERWRITE
|
00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder]
HDFS partitions=1/1 files=1 size=1.08KB
+ Iceberg snapshot id: 7350750578864730166
row-size=36B cardinality=1
---- DISTRIBUTEDPLAN
WRITE TO HDFS [functional_parquet.iceberg_partition_transforms_zorder, OVERWRITE=true, PARTITION-KEYS=(year(functional_parquet.iceberg_partition_transforms_zorder.ts),iceberg_bucket_transform(functional_parquet.iceberg_partition_transforms_zorder.s, 5))]
@@ -413,5 +414,6 @@ WRITE TO HDFS [functional_parquet.iceberg_partition_transforms_zorder, OVERWRITE
|
00:SCAN HDFS [functional_parquet.iceberg_partition_transforms_zorder]
HDFS partitions=1/1 files=1 size=1.08KB
+ Iceberg snapshot id: 7350750578864730166
row-size=36B cardinality=1
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
index 795e7ff79..6c3b986f5 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
@@ -258,6 +258,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_non_partitioned]
HDFS partitions=1/1 files=3 size=3.41KB
+ Iceberg snapshot id: 93996984692289973
stored statistics:
table: rows=20 size=22.90KB
columns: unavailable
@@ -283,6 +284,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_non_partitioned]
HDFS partitions=1/1 files=3 size=3.41KB
+ Iceberg snapshot id: 93996984692289973
stored statistics:
table: rows=20 size=22.90KB
columns: all
@@ -302,6 +304,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=10 size=11.46KB
+ Iceberg snapshot id: 8270633197658268308
stored statistics:
table: rows=20 size=22.90KB
columns: unavailable
@@ -324,6 +327,7 @@ PLAN-ROOT SINK
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
HDFS partitions=1/1 files=4 size=4.57KB
predicates: id > CAST(0 AS INT)
+ Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: action = 'click'
stored statistics:
table: rows=20 size=22.90KB
@@ -349,17 +353,18 @@ PLAN-ROOT SINK
04:UNION
| pass-through-operands: all
| mem-estimate=0B mem-reservation=0B thread-reservation=0
-| tuple-ids=0 row-size=36B cardinality=4
+| tuple-ids=0 row-size=36B cardinality=2
| in pipelines: 03(GETNEXT), 00(GETNEXT)
|
|--02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN]
| | equality predicates: functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.file__position = functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete.pos, functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files.input__file__name = functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete.file_path
| | mem-estimate=566B mem-reservation=566B thread-reservation=0
-| | tuple-ids=0 row-size=36B cardinality=3
+| | tuple-ids=0 row-size=36B cardinality=1
| | in pipelines: 00(GETNEXT), 01(OPEN)
| |
| |--01:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-POSITION-DELETE-01 functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files-position-delete]
| | HDFS partitions=1/1 files=2 size=5.33KB
+| | Iceberg snapshot id: 1497619269847778439
| | stored statistics:
| | table: rows=4 size=5.33KB
| | columns: all
@@ -370,6 +375,7 @@ PLAN-ROOT SINK
| |
| 00:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
| HDFS partitions=1/1 files=1 size=625B
+| Iceberg snapshot id: 1497619269847778439
| stored statistics:
| table: rows=10 size=7.77KB
| columns missing stats: i, s
@@ -380,6 +386,7 @@ PLAN-ROOT SINK
|
03:SCAN HDFS [functional_parquet.iceberg_v2_positional_not_all_data_files_have_delete_files]
HDFS partitions=1/1 files=1 size=620B
+ Iceberg snapshot id: 1497619269847778439
stored statistics:
table: rows=10 size=7.77KB
columns missing stats: i, s
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index 8679a3e91..ed34262a6 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -462,6 +462,12 @@ class TestIcebergTable(IcebergTestSuite):
"select count(*) from {0} for system_time as of {1}".format(tbl_name, ts),
expected)
+ def expect_snapshot_id_in_plan_t(ts, snapshot_id):
+ data = impalad_client.execute(
+ "explain select * from {0} for system_time as of {1}".format(
+ tbl_name, ts))
+ assert " Iceberg snapshot id: {0}".format(snapshot_id) in data.data
+
def expect_results_v(snapshot_id, expected_results, expected_cols):
expect_results(
"select * from {0} for system_version as of {1}".format(tbl_name, snapshot_id),
@@ -473,6 +479,13 @@ class TestIcebergTable(IcebergTestSuite):
tbl_name, snapshot_id),
expected)
+ def expect_snapshot_id_in_plan_v(snapshot_id):
+ data = impalad_client.execute(
+ "explain select * from {0} for system_version as of {1}".format(
+ tbl_name, snapshot_id))
+ assert " Iceberg snapshot id: {0}".format(snapshot_id) in data.data
+
+
def impala_now():
now_data = impalad_client.execute("select now()")
return now_data.data[0]
@@ -540,6 +553,18 @@ class TestIcebergTable(IcebergTestSuite):
expect_results_v(snapshots[3].get_snapshot_id(), ['100'], i_cols)
expect_results_v(snapshots[4].get_snapshot_id(), ['100\tNULL', '3\t103'], ij_cols)
+ expect_snapshot_id_in_plan_v(snapshots[0].get_snapshot_id())
+ expect_snapshot_id_in_plan_v(snapshots[1].get_snapshot_id())
+ expect_snapshot_id_in_plan_v(snapshots[2].get_snapshot_id())
+ expect_snapshot_id_in_plan_v(snapshots[3].get_snapshot_id())
+ expect_snapshot_id_in_plan_v(snapshots[4].get_snapshot_id())
+
+ expect_snapshot_id_in_plan_t(quote(ts_1), snapshots[0].get_snapshot_id())
+ expect_snapshot_id_in_plan_t(quote(ts_2), snapshots[1].get_snapshot_id())
+ expect_snapshot_id_in_plan_t(quote(ts_3), snapshots[2].get_snapshot_id())
+ expect_snapshot_id_in_plan_t(quote(ts_4), snapshots[3].get_snapshot_id())
+ expect_snapshot_id_in_plan_t(quote(ts_5), snapshots[4].get_snapshot_id())
+
# Test of plain count star optimization
# 'NumRowGroups' and 'NumFileMetadataRead' should not appear in profile
expect_for_count_star_t("now()", '2')
(impala) 01/02: IMPALA-12378: Auto Ship JDBC Data Source
Posted by wz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit fc74ca672a99c4d473a48398bd426f7f69df833c
Author: wzhou-code <wz...@cloudera.com>
AuthorDate: Sat Feb 3 22:19:03 2024 -0800
IMPALA-12378: Auto Ship JDBC Data Source
This patch moves the source files of jdbc package to fe.
Data source location is optional. Data source could be created without
specifying HDFS location. Assume data source class is in the classpath
and instance of data source class could be created with current class
loader. Impala still try to load the jar file of the data source in
runtime if it's set in data source location.
Testing:
- Passed core test
- Passed dockerised-tests
Change-Id: I0daff8db6231f161ec27b45b51d78e21733d9b1f
Reviewed-on: http://gerrit.cloudera.org:8080/20971
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Wenzhe Zhou <wz...@cloudera.com>
---
be/src/exec/external-data-source-executor.cc | 6 ++--
bin/impala-config.sh | 1 +
bin/rat_exclude_files.txt | 2 +-
fe/pom.xml | 6 ++++
.../org/apache/impala/compat/MetastoreShim.java | 6 ++--
fe/src/main/cup/sql-parser.cup | 4 +--
.../apache/impala/analysis/CreateDataSrcStmt.java | 11 ++++---
.../impala/analysis/CreateTableDataSrcStmt.java | 8 +++--
.../extdatasource/ExternalDataSourceExecutor.java | 19 +++++++----
.../impala/extdatasource/jdbc/JdbcDataSource.java | 0
.../org/apache/impala/extdatasource/jdbc/README.md | 0
.../extdatasource/jdbc/conf/DatabaseType.java | 0
.../extdatasource/jdbc/conf/JdbcStorageConfig.java | 0
.../jdbc/conf/JdbcStorageConfigManager.java | 0
.../jdbc/dao/DB2DatabaseAccessor.java | 0
.../extdatasource/jdbc/dao/DatabaseAccessor.java | 0
.../jdbc/dao/DatabaseAccessorFactory.java | 0
.../jdbc/dao/GenericJdbcDatabaseAccessor.java | 7 ++--
.../jdbc/dao/ImpalaDatabaseAccessor.java | 0
.../extdatasource/jdbc/dao/JdbcRecordIterator.java | 0
.../jdbc/dao/JethroDatabaseAccessor.java | 0
.../jdbc/dao/MsSqlDatabaseAccessor.java | 0
.../jdbc/dao/MySqlDatabaseAccessor.java | 0
.../jdbc/dao/OracleDatabaseAccessor.java | 0
.../jdbc/dao/PostgresDatabaseAccessor.java | 0
.../exception/JdbcDatabaseAccessException.java | 0
.../jdbc/util/QueryConditionUtil.java | 0
.../apache/impala/planner/DataSourceScanNode.java | 37 +++++++++++++++-------
.../org/apache/impala/analysis/AnalyzeDDLTest.java | 1 +
.../org/apache/impala/analysis/ParserTest.java | 3 +-
.../apache/impala/customcluster/LdapHS2Test.java | 3 +-
java/pom.xml | 1 +
testdata/bin/copy-ext-data-sources.sh | 7 ----
.../impala-ext-jdbc-tables-predicates.test | 3 +-
.../queries/QueryTest/impala-ext-jdbc-tables.test | 3 +-
.../QueryTest/jdbc-data-source-with-keystore.test | 3 +-
.../queries/QueryTest/jdbc-data-source.test | 3 +-
.../queries/QueryTest/mysql-ext-jdbc-tables.test | 3 +-
38 files changed, 79 insertions(+), 58 deletions(-)
diff --git a/be/src/exec/external-data-source-executor.cc b/be/src/exec/external-data-source-executor.cc
index 34ae9e666..3041609f8 100644
--- a/be/src/exec/external-data-source-executor.cc
+++ b/be/src/exec/external-data-source-executor.cc
@@ -139,8 +139,10 @@ Status ExternalDataSourceExecutor::Init(const string& jar_path,
string local_jar_path;
// TODO(IMPALA-6727): pass the mtime from the coordinator. for now, skip the mtime
// check (-1).
- RETURN_IF_ERROR(LibCache::instance()->GetLocalPath(
- jar_path, LibCache::TYPE_JAR, -1, &handle, &local_jar_path));
+ if (!jar_path.empty()) {
+ RETURN_IF_ERROR(LibCache::instance()->GetLocalPath(
+ jar_path, LibCache::TYPE_JAR, -1, &handle, &local_jar_path));
+ }
JNIEnv* jni_env = JniUtil::GetJNIEnv();
// Add a scoped cleanup jni reference object. This cleans up local refs made below.
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 0886006bd..4543bc169 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -290,6 +290,7 @@ export IMPALA_SLF4J_VERSION=2.0.3
export IMPALA_SPRINGFRAMEWORK_VERSION=5.3.27
export IMPALA_XMLSEC_VERSION=2.2.3
export IMPALA_OBS_VERSION=3.1.1-hw-42
+export IMPALA_DBCP2_VERSION=2.9.0
# When Impala is building docker images on Redhat-based distributions,
# it is useful to be able to customize the base image. Some users will
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 43d5df250..f93ffdc0a 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -131,6 +131,7 @@ testdata/*.test
be/src/kudu/util/testdata/*.txt
be/src/testutil/*.pem
*.json
+fe/src/main/java/org/apache/impala/extdatasource/jdbc/README.md
fe/src/test/resources/*.xml
fe/src/test/resources/adschema.ldif
fe/src/test/resources/adusers.ldif
@@ -138,7 +139,6 @@ fe/src/test/resources/hbase-jaas-client.conf.template
fe/src/test/resources/hbase-jaas-server.conf.template
fe/src/test/resources/users.ldif
java/.mvn/maven.config
-java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/README.md
java/toolchains.xml.tmpl
testdata/AllTypesError/*.txt
testdata/AllTypesErrorNoNulls/*.txt
diff --git a/fe/pom.xml b/fe/pom.xml
index 26232bc17..85ad37676 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -344,6 +344,12 @@ under the License.
<version>${derby.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
+ <version>${commons-dbcp2.version}</version>
+ </dependency>
+
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
diff --git a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index a1ea42fe8..516fe6b7a 100644
--- a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -1041,18 +1041,18 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
* Convert DataConnector object to DataSource object.
*/
private static DataSource dataConnectorToDataSource(DataConnector connector) {
- if (!connector.isSetName() || !connector.isSetType() || !connector.isSetUrl()
+ if (!connector.isSetName() || !connector.isSetType()
|| !connector.isSetDescription() || connector.getParametersSize() == 0
|| !connector.getType().equalsIgnoreCase(HMS_DATA_CONNECTOR_TYPE)) {
return null;
}
String name = connector.getName();
- String location = connector.getUrl();
+ String location = connector.isSetUrl() ? connector.getUrl() : "";
String className =
connector.getParameters().get(HMS_DATA_CONNECTOR_PARAM_KEY_CLASS_NAME);
String apiVersion =
connector.getParameters().get(HMS_DATA_CONNECTOR_PARAM_KEY_API_VERSION);
- if (!Strings.isNullOrEmpty(name) && !Strings.isNullOrEmpty(location) &&
+ if (!Strings.isNullOrEmpty(name) && location != null &&
!Strings.isNullOrEmpty(className) && !Strings.isNullOrEmpty(apiVersion)) {
return new DataSource(name, location, className, apiVersion);
}
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index af596629c..905e7bb23 100755
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -2326,11 +2326,11 @@ create_view_stmt ::=
create_data_src_stmt ::=
KW_CREATE KW_DATA source_ident:is_source_id
if_not_exists_val:if_not_exists ident_or_default:data_src_name
- KW_LOCATION STRING_LITERAL:location
+ location_val:location
KW_CLASS STRING_LITERAL:class_name
KW_API_VERSION STRING_LITERAL:api_version
{:
- RESULT = new CreateDataSrcStmt(data_src_name, new HdfsUri(location), class_name,
+ RESULT = new CreateDataSrcStmt(data_src_name, location, class_name,
api_version, if_not_exists);
:}
;
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateDataSrcStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateDataSrcStmt.java
index b2523f67d..55301019c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateDataSrcStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateDataSrcStmt.java
@@ -44,7 +44,6 @@ public class CreateDataSrcStmt extends StatementBase {
Preconditions.checkNotNull(dataSrcName);
Preconditions.checkNotNull(className);
Preconditions.checkNotNull(apiVersionString);
- Preconditions.checkNotNull(location);
dataSrcName_ = dataSrcName.toLowerCase();
location_ = location;
className_ = className;
@@ -68,7 +67,9 @@ public class CreateDataSrcStmt extends StatementBase {
"'. Valid API versions: " + Joiner.on(", ").join(ApiVersion.values()));
}
- location_.analyze(analyzer, Privilege.ALL, FsAction.READ);
+ if (location_ != null) {
+ location_.analyze(analyzer, Privilege.ALL, FsAction.READ);
+ }
// TODO: Check class exists and implements API version
// TODO: authorization check
}
@@ -80,7 +81,7 @@ public class CreateDataSrcStmt extends StatementBase {
if (ifNotExists_) sb.append("IF NOT EXISTS ");
sb.append(dataSrcName_);
sb.append(" LOCATION '");
- sb.append(location_.getLocation());
+ sb.append(location_ != null ? location_.getLocation() : "");
sb.append("' CLASS '");
sb.append(className_);
sb.append("' API_VERSION '");
@@ -91,7 +92,7 @@ public class CreateDataSrcStmt extends StatementBase {
public TCreateDataSourceParams toThrift() {
return new TCreateDataSourceParams(
- new TDataSource(dataSrcName_, location_.toString(), className_,
- apiVersion_.name())).setIf_not_exists(ifNotExists_);
+ new TDataSource(dataSrcName_, location_ != null ? location_.toString() : "",
+ className_, apiVersion_.name())).setIf_not_exists(ifNotExists_);
}
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableDataSrcStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableDataSrcStmt.java
index 13eda5a7b..61c9af50c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableDataSrcStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableDataSrcStmt.java
@@ -62,14 +62,16 @@ public class CreateTableDataSrcStmt extends CreateTableStmt {
"not support the column type: " + col.getType());
}
}
+
// Add table properties from the DataSource catalog object now that we have access
// to the catalog. These are stored in the table metadata so that the table could
// be scanned without the DataSource catalog object.
String location = dataSource.getLocation();
- getTblProperties().put(TBL_PROP_LOCATION, location);
+ getTblProperties().put(TBL_PROP_LOCATION, location != null ? location : "");
getTblProperties().put(TBL_PROP_CLASS, dataSource.getClassName());
getTblProperties().put(TBL_PROP_API_VER, dataSource.getApiVersion());
- new HdfsUri(location).analyze(analyzer, Privilege.ALL, FsAction.READ);
- // TODO: check class exists and implements API version
+ if (!Strings.isNullOrEmpty(location)) {
+ new HdfsUri(location).analyze(analyzer, Privilege.ALL, FsAction.READ);
+ }
}
}
diff --git a/fe/src/main/java/org/apache/impala/extdatasource/ExternalDataSourceExecutor.java b/fe/src/main/java/org/apache/impala/extdatasource/ExternalDataSourceExecutor.java
index ff041314c..8545567b0 100644
--- a/fe/src/main/java/org/apache/impala/extdatasource/ExternalDataSourceExecutor.java
+++ b/fe/src/main/java/org/apache/impala/extdatasource/ExternalDataSourceExecutor.java
@@ -47,6 +47,7 @@ import org.apache.impala.extdatasource.v1.ExternalDataSource;
import org.apache.impala.thrift.TErrorCode;
import org.apache.impala.thrift.TStatus;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -107,15 +108,14 @@ public class ExternalDataSourceExecutor {
/**
* @param jarPath The local path to the jar containing the ExternalDataSource.
+ * It is null or empty if the jar file of data source is already in classpath.
* @param className The name of the class implementing the ExternalDataSource.
* @param apiVersionStr The API version the ExternalDataSource implements.
- * Must be a valid value of {@link ApiVersion}.
+ * Must be a valid value of {@link ApiVersion}.
* @param initString The init string registered with this data source.
*/
public ExternalDataSourceExecutor(String jarPath, String className,
String apiVersionStr, String initString) throws ImpalaException {
- Preconditions.checkNotNull(jarPath);
-
apiVersion_ = ApiVersion.valueOf(apiVersionStr);
if (apiVersion_ == null) {
throw new ImpalaRuntimeException("Invalid API version: " + apiVersionStr);
@@ -130,8 +130,8 @@ public class ExternalDataSourceExecutor {
dataSource_ = (ExternalDataSource) ctor.newInstance();
} catch (Exception ex) {
throw new ImpalaRuntimeException(String.format("Unable to load external data " +
- "source library from path=%s className=%s apiVersion=%s", jarPath,
- className, apiVersionStr), ex);
+ "source library from path=%s className=%s apiVersion=%s",
+ jarPath != null ? jarPath : "Impala classpath", className, apiVersionStr), ex);
}
}
@@ -141,6 +141,11 @@ public class ExternalDataSourceExecutor {
*/
private Class<?> getDataSourceClass() throws Exception {
Class<?> c = null;
+ if (Strings.isNullOrEmpty(jarPath_)) {
+ c = Class.forName(className_);
+ LOG.trace("Get instance of DataSourceClass in current ClassLoader");
+ return c;
+ }
// Cache map key needs to contain both the class name and init string in case
// the same class is used for multiple tables where some are cached and others
// are not.
@@ -248,8 +253,8 @@ public class ExternalDataSourceExecutor {
}
String errorMessage = String.format(
"Error in data source (path=%s, class=%s, version=%s) %s: %s",
- jarPath_, className_, apiVersion_.name(), opName,
- exceptionMessage);
+ jarPath_ != null ? jarPath_ : "Impala classpath", className_, apiVersion_.name(),
+ opName, exceptionMessage);
LOG.error(errorMessage, e); // Logs the stack
return new TStatus(TErrorCode.RUNTIME_ERROR, Lists.newArrayList(errorMessage));
}
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/JdbcDataSource.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/JdbcDataSource.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/JdbcDataSource.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/JdbcDataSource.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/README.md b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/README.md
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/README.md
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/README.md
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/conf/DatabaseType.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/DatabaseType.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/conf/DatabaseType.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/DatabaseType.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfig.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfig.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfig.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfig.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/conf/JdbcStorageConfigManager.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DB2DatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DB2DatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DB2DatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DB2DatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessorFactory.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessorFactory.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessorFactory.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/DatabaseAccessorFactory.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/GenericJdbcDatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/GenericJdbcDatabaseAccessor.java
similarity index 98%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/GenericJdbcDatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/GenericJdbcDatabaseAccessor.java
index 407135157..f071ee173 100644
--- a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/GenericJdbcDatabaseAccessor.java
+++ b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/GenericJdbcDatabaseAccessor.java
@@ -76,7 +76,7 @@ public class GenericJdbcDatabaseAccessor implements DatabaseAccessor {
protected DataSource dbcpDataSource = null;
// Cache datasource for sharing
- public static Cache<String, DataSource> dataSourceCache = CacheBuilder
+ protected static final Cache<String, DataSource> dataSourceCache = CacheBuilder
.newBuilder()
.removalListener((RemovalListener<String, DataSource>) notification -> {
DataSource ds = notification.getValue();
@@ -164,9 +164,9 @@ public class GenericJdbcDatabaseAccessor implements DatabaseAccessor {
@Override
public void close(boolean cleanCache) {
dbcpDataSource = null;
- if (cleanCache && dataSourceCache != null) {
+ if (cleanCache) {
+ Preconditions.checkNotNull(dataSourceCache);
dataSourceCache.invalidateAll();
- dataSourceCache = null;
}
if (jdbcDriverLocalPath != null) {
// Delete the jar file of jdbc driver.
@@ -287,6 +287,7 @@ public class GenericJdbcDatabaseAccessor implements DatabaseAccessor {
String jdbcUrl = props.getProperty("url");
String username = props.getProperty("username", "-");
String cacheMapKey = String.format("%s.%s", jdbcUrl, username);
+ Preconditions.checkNotNull(dataSourceCache);
dbcpDataSource = dataSourceCache.get(cacheMapKey,
() -> {
LOG.info("Datasource for '{}' was not cached. "
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/ImpalaDatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/ImpalaDatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/ImpalaDatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/ImpalaDatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JdbcRecordIterator.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JdbcRecordIterator.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JdbcRecordIterator.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JdbcRecordIterator.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JethroDatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JethroDatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JethroDatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JethroDatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MsSqlDatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MsSqlDatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MsSqlDatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MsSqlDatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MySqlDatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MySqlDatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MySqlDatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/MySqlDatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/OracleDatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/OracleDatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/OracleDatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/OracleDatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/PostgresDatabaseAccessor.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/PostgresDatabaseAccessor.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/dao/PostgresDatabaseAccessor.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/PostgresDatabaseAccessor.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/exception/JdbcDatabaseAccessException.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/exception/JdbcDatabaseAccessException.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/exception/JdbcDatabaseAccessException.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/exception/JdbcDatabaseAccessException.java
diff --git a/java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/util/QueryConditionUtil.java b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/util/QueryConditionUtil.java
similarity index 100%
rename from java/ext-data-source/jdbc/src/main/java/org/apache/impala/extdatasource/jdbc/util/QueryConditionUtil.java
rename to fe/src/main/java/org/apache/impala/extdatasource/jdbc/util/QueryConditionUtil.java
diff --git a/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java b/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
index 408789cb7..c1fc8ab5e 100644
--- a/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
@@ -65,6 +65,7 @@ import org.apache.impala.thrift.TStatus;
import com.google.common.base.Joiner;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -169,16 +170,28 @@ public class DataSourceScanNode extends ScanNode {
}
}
- String dsLocation = table_.getDataSource().getHdfs_location();
- String localPath;
- try {
- localPath = FileSystemUtil.copyFileFromUriToLocal(dsLocation);
- } catch (IOException e) {
- throw new InternalException(String.format(
- "Unable to fetch data source jar from location '%s'.", dsLocation));
- }
String className = table_.getDataSource().getClass_name();
String apiVersion = table_.getDataSource().getApi_version();
+ String dsLocation = table_.getDataSource().getHdfs_location();
+ // Check if the data source jar file exists in the HDFS location. If yes, then copy
+ // the jar file to the local file system, and get the instance of the data source
+ // class in URLClassLoader which is generated from copied local jar file. Otherwise
+ // assume the jar file is in the classpath and try to get the instance of the data
+ // source class in current ClassLoader.
+ String localPath = null;
+ if (!Strings.isNullOrEmpty(dsLocation)) {
+ LOG.trace("Get the instance from URLClassLoader");
+ try {
+ localPath = FileSystemUtil.copyFileFromUriToLocal(dsLocation);
+ } catch (IOException e) {
+ throw new InternalException(String.format(
+ "Unable to fetch data source jar from location '%s'.", dsLocation));
+ }
+ Preconditions.checkNotNull(localPath);
+ } else {
+ LOG.trace("Get the instance of the data source class in current ClassLoader");
+ }
+
TPrepareResult prepareResult;
TStatus prepareStatus;
try {
@@ -196,9 +209,11 @@ public class DataSourceScanNode extends ScanNode {
"Error calling prepare() on data source %s",
DataSource.debugString(table_.getDataSource())), e);
} finally {
- // Delete the jar file once its loaded
- Path localJarPath = new Path("file://" + localPath);
- FileSystemUtil.deleteIfExists(localJarPath);
+ if (!Strings.isNullOrEmpty(localPath)) {
+ // Delete the jar file once its loaded
+ Path localJarPath = new Path("file://" + localPath);
+ FileSystemUtil.deleteIfExists(localJarPath);
+ }
}
if (prepareStatus.getStatus_code() != TErrorCode.OK) {
throw new InternalException(String.format(
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index 1976a418c..b4424097c 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -2086,6 +2086,7 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"CLASS 'com.bar.Foo' API_VERSION 'V1'");
AnalyzesOk("CREATE DATA SOURCE foo LOCATION 's3a://bucket/a/b/foo.jar' " +
"CLASS 'com.bar.Foo' API_VERSION 'V1'");
+ AnalyzesOk("CREATE DATA SOURCE foo CLASS 'com.bar.Foo' API_VERSION 'V1'");
AnalysisError("CREATE DATA SOURCE foo LOCATION 'blah://localhost:20500/foo.jar' " +
"CLASS 'com.bar.Foo' API_VERSION 'V1'",
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index 3bd6fda9e..9d9397b59 100755
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -3115,6 +3115,7 @@ public class ParserTest extends FrontendTestBase {
"API_VERSION \"V1\"");
ParsesOk("CREATE DATA SOURCE foo LOCATION '/x/foo@hi_^!#.jar' CLASS 'com.bar.Foo' " +
"API_VERSION 'V1'");
+ ParsesOk("CREATE DATA SOURCE foo CLASS 'com.bar.Foo' API_VERSION 'V1'");
ParserError("CREATE DATA foo LOCATION '/foo.jar' CLASS 'com.bar.Foo' " +
"API_VERSION 'V1'");
@@ -3130,8 +3131,6 @@ public class ParserTest extends FrontendTestBase {
"API_VERSION V1");
ParserError("CREATE DATA SOURCE LOCATION '/x/foo.jar' CLASS 'com.bar.Foo' " +
"API_VERSION 'V1'");
- ParserError("CREATE DATA SOURCE foo CLASS 'com.bar.Foo' " +
- "API_VERSION 'V1'");
ParserError("CREATE DATA SOURCE foo LOCATION CLASS 'com.bar.Foo' " +
"API_VERSION 'V1'");
ParserError("CREATE DATA SOURCE foo LOCATION '/foo.jar' API_VERSION 'V1'");
diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java b/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java
index 0d25968ba..4e5d012fa 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java
@@ -741,9 +741,8 @@ public class LdapHS2Test {
String dropDSQuery = "DROP DATA SOURCE IF EXISTS impala_jdbc_test_ds";
String createDSQuery = String.format("CREATE DATA SOURCE impala_jdbc_test_ds " +
- "LOCATION '%s/test-warehouse/data-sources/jdbc-data-source.jar' " +
"CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource' " +
- "API_VERSION 'V1'", fileSystemPrefix);
+ "API_VERSION 'V1'");
String dropTableQuery = "DROP TABLE IF EXISTS %s";
// Set JDBC authentication mechanisms as LDAP (3) with username/password as
// TEST_USER_1/TEST_PASSWORD_1.
diff --git a/java/pom.xml b/java/pom.xml
index 0116b6b8f..e7a2d3d72 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -79,6 +79,7 @@ under the License.
<bouncy-castle.version>${env.IMPALA_BOUNCY_CASTLE_VERSION}</bouncy-castle.version>
<springframework.version>${env.IMPALA_SPRINGFRAMEWORK_VERSION}</springframework.version>
<json-smart.version>${env.IMPALA_JSON_SMART_VERSION}</json-smart.version>
+ <commons-dbcp2.version>${env.IMPALA_DBCP2_VERSION}</commons-dbcp2.version>
</properties>
<repositories>
diff --git a/testdata/bin/copy-ext-data-sources.sh b/testdata/bin/copy-ext-data-sources.sh
index 8e0f023e2..4f33a4a46 100755
--- a/testdata/bin/copy-ext-data-sources.sh
+++ b/testdata/bin/copy-ext-data-sources.sh
@@ -40,13 +40,6 @@ hadoop fs -put -f \
echo "Copied" ${EXT_DATA_SOURCE_SRC_PATH}/test/target/impala-data-source-test-*.jar \
"into HDFS" ${EXT_DATA_SOURCES_HDFS_PATH}
-hadoop fs -put -f \
- ${EXT_DATA_SOURCE_SRC_PATH}/jdbc/target/impala-data-source-jdbc-*.jar \
- ${EXT_DATA_SOURCES_HDFS_PATH}/jdbc-data-source.jar
-
-echo "Copied" ${EXT_DATA_SOURCE_SRC_PATH}/jdbc/target/impala-data-source-jdbc-*.jar \
- "into HDFS" ${EXT_DATA_SOURCES_HDFS_PATH}
-
# Copy Postgres JDBC driver to HDFS
hadoop fs -put -f \
${IMPALA_HOME}/fe/target/dependency/postgresql-*.jar \
diff --git a/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables-predicates.test b/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables-predicates.test
index f46b6a75f..ec8c7c638 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables-predicates.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables-predicates.test
@@ -3,7 +3,6 @@
# Create DataSource
DROP DATA SOURCE IF EXISTS TestJdbcDataSource;
CREATE DATA SOURCE TestJdbcDataSource
-LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-data-source.jar'
CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource'
API_VERSION 'V1';
---- RESULTS
@@ -15,7 +14,7 @@ SHOW DATA SOURCES LIKE 'testjdbcdatasource';
---- LABELS
NAME,LOCATION,CLASS NAME,API VERSION
---- RESULTS
-'testjdbcdatasource',regex:'.*/test-warehouse/data-sources/jdbc-data-source.jar','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
+'testjdbcdatasource','','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
---- TYPES
STRING,STRING,STRING,STRING
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables.test b/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables.test
index b68db24f5..9ca4b0909 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables.test
@@ -3,7 +3,6 @@
# Create DataSource
DROP DATA SOURCE IF EXISTS TestJdbcDataSource;
CREATE DATA SOURCE TestJdbcDataSource
-LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-data-source.jar'
CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource'
API_VERSION 'V1';
---- RESULTS
@@ -15,7 +14,7 @@ SHOW DATA SOURCES LIKE 'testjdbcdatasource';
---- LABELS
NAME,LOCATION,CLASS NAME,API VERSION
---- RESULTS
-'testjdbcdatasource',regex:'.*/test-warehouse/data-sources/jdbc-data-source.jar','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
+'testjdbcdatasource','','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
---- TYPES
STRING,STRING,STRING,STRING
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source-with-keystore.test b/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source-with-keystore.test
index 2e961d9a3..7e1d1bcd4 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source-with-keystore.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source-with-keystore.test
@@ -3,7 +3,6 @@
# Create DataSource
DROP DATA SOURCE IF EXISTS TestJdbcDataSourceWithKeystore;
CREATE DATA SOURCE TestJdbcDataSourceWithKeystore
-LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-data-source.jar'
CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource'
API_VERSION 'V1';
---- RESULTS
@@ -15,7 +14,7 @@ SHOW DATA SOURCES LIKE 'testjdbcdatasourcewithkeystore';
---- LABELS
NAME,LOCATION,CLASS NAME,API VERSION
---- RESULTS
-'testjdbcdatasourcewithkeystore',regex:'.*/test-warehouse/data-sources/jdbc-data-source.jar','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
+'testjdbcdatasourcewithkeystore','','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
---- TYPES
STRING,STRING,STRING,STRING
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source.test b/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source.test
index 9df510fc3..dd8d32cd2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source.test
@@ -3,7 +3,6 @@
# Create DataSource
DROP DATA SOURCE IF EXISTS TestJdbcDataSource;
CREATE DATA SOURCE TestJdbcDataSource
-LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-data-source.jar'
CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource'
API_VERSION 'V1';
---- RESULTS
@@ -15,7 +14,7 @@ SHOW DATA SOURCES LIKE 'testjdbcdatasource';
---- LABELS
NAME,LOCATION,CLASS NAME,API VERSION
---- RESULTS
-'testjdbcdatasource',regex:'.*/test-warehouse/data-sources/jdbc-data-source.jar','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
+'testjdbcdatasource','','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
---- TYPES
STRING,STRING,STRING,STRING
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/mysql-ext-jdbc-tables.test b/testdata/workloads/functional-query/queries/QueryTest/mysql-ext-jdbc-tables.test
index 0d96ee5e7..172719348 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/mysql-ext-jdbc-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/mysql-ext-jdbc-tables.test
@@ -3,7 +3,6 @@
# Create DataSource
DROP DATA SOURCE IF EXISTS TestJdbcDataSource;
CREATE DATA SOURCE TestJdbcDataSource
-LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-data-source.jar'
CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource'
API_VERSION 'V1';
---- RESULTS
@@ -15,7 +14,7 @@ SHOW DATA SOURCES LIKE 'testjdbcdatasource';
---- LABELS
NAME,LOCATION,CLASS NAME,API VERSION
---- RESULTS
-'testjdbcdatasource',regex:'.*/test-warehouse/data-sources/jdbc-data-source.jar','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
+'testjdbcdatasource','','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
---- TYPES
STRING,STRING,STRING,STRING
====