You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2025/11/03 03:55:32 UTC
(iceberg) branch 1.10.x updated (4119a36eda -> 16f5bc6934)
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a change to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
from 4119a36eda Parquet, Data, Spark: Fix variant type filtering in ParquetMetricsRowGroupFilter (#14081) (#14467)
new 0462b03cc3 Core: Handle unpartitioned check for a spec with all void transforms in replace partitions (#14186)
new cc51275492 AWS: exclude logging classes from bundle (#14225)
new 16f5bc6934 BigQuery: Add iceberg-bigquery dependency to spark and flink build scripts (#14221)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
aws-bundle/build.gradle | 4 +-
build.gradle | 1 +
.../org/apache/iceberg/BaseReplacePartitions.java | 2 +-
.../org/apache/iceberg/TestReplacePartitions.java | 53 ++++++++++++++++++++++
flink/v1.19/build.gradle | 1 +
flink/v1.20/build.gradle | 1 +
flink/v2.0/build.gradle | 1 +
spark/v3.4/build.gradle | 1 +
spark/v3.5/build.gradle | 1 +
spark/v4.0/build.gradle | 1 +
10 files changed, 64 insertions(+), 2 deletions(-)
(iceberg) 03/03: BigQuery: Add iceberg-bigquery dependency to spark and flink build scripts (#14221)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 16f5bc6934e2a7d172936b679184edc27fc7fa29
Author: Владислав Самороков <33...@users.noreply.github.com>
AuthorDate: Wed Oct 1 13:34:43 2025 +0700
BigQuery: Add iceberg-bigquery dependency to spark and flink build scripts (#14221)
(cherry picked from commit 2b9a7e3fd42c8ab82d053760bced09cdbf82a1cf)
---
build.gradle | 1 +
flink/v1.19/build.gradle | 1 +
flink/v1.20/build.gradle | 1 +
flink/v2.0/build.gradle | 1 +
spark/v3.4/build.gradle | 1 +
spark/v3.5/build.gradle | 1 +
spark/v4.0/build.gradle | 1 +
7 files changed, 7 insertions(+)
diff --git a/build.gradle b/build.gradle
index 6bc052885f..ccd05f67e6 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1037,6 +1037,7 @@ project(':iceberg-open-api') {
testFixturesImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testFixturesImplementation project(':iceberg-aws')
testFixturesImplementation project(':iceberg-gcp')
+ testFixturesImplementation project(':iceberg-bigquery')
testFixturesImplementation project(':iceberg-azure')
testFixturesImplementation(libs.hadoop3.common) {
exclude group: 'org.slf4j'
diff --git a/flink/v1.19/build.gradle b/flink/v1.19/build.gradle
index 05fdddd63c..26c53e1518 100644
--- a/flink/v1.19/build.gradle
+++ b/flink/v1.19/build.gradle
@@ -167,6 +167,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
}
diff --git a/flink/v1.20/build.gradle b/flink/v1.20/build.gradle
index 16d1d76ec1..d0bbe7a898 100644
--- a/flink/v1.20/build.gradle
+++ b/flink/v1.20/build.gradle
@@ -167,6 +167,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
}
diff --git a/flink/v2.0/build.gradle b/flink/v2.0/build.gradle
index dfbaa8ff41..01fbc1a4fe 100644
--- a/flink/v2.0/build.gradle
+++ b/flink/v2.0/build.gradle
@@ -167,6 +167,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
}
diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle
index 714be0831d..69baf216c3 100644
--- a/spark/v3.4/build.gradle
+++ b/spark/v3.4/build.gradle
@@ -246,6 +246,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
implementation project(':iceberg-hive-metastore')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle
index 69700d8436..42eb35c3b4 100644
--- a/spark/v3.5/build.gradle
+++ b/spark/v3.5/build.gradle
@@ -246,6 +246,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
implementation project(':iceberg-hive-metastore')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle
index 9c7ea06f99..237337707e 100644
--- a/spark/v4.0/build.gradle
+++ b/spark/v4.0/build.gradle
@@ -251,6 +251,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
implementation project(':iceberg-hive-metastore')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
(iceberg) 01/03: Core: Handle unpartitioned check for a spec with all void transforms in replace partitions (#14186)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 0462b03cc354cef7db256cd6543873b3415edae7
Author: Yubo Xu <yu...@gmail.com>
AuthorDate: Tue Sep 30 08:50:19 2025 -0700
Core: Handle unpartitioned check for a spec with all void transforms in replace partitions (#14186)
(cherry picked from commit 00d18e38feb5439f715ee3c13e206e9445ed2faa)
---
.../org/apache/iceberg/BaseReplacePartitions.java | 2 +-
.../org/apache/iceberg/TestReplacePartitions.java | 53 ++++++++++++++++++++++
2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java b/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java
index 04611f50f1..892257b51b 100644
--- a/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java
+++ b/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java
@@ -111,7 +111,7 @@ public class BaseReplacePartitions extends MergingSnapshotProducer<ReplacePartit
@Override
public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
- if (dataSpec().fields().isEmpty()) {
+ if (dataSpec().isUnpartitioned()) {
// replace all data in an unpartitioned table
deleteByRowFilter(Expressions.alwaysTrue());
}
diff --git a/core/src/test/java/org/apache/iceberg/TestReplacePartitions.java b/core/src/test/java/org/apache/iceberg/TestReplacePartitions.java
index 29daeb995c..abc27cddd6 100644
--- a/core/src/test/java/org/apache/iceberg/TestReplacePartitions.java
+++ b/core/src/test/java/org/apache/iceberg/TestReplacePartitions.java
@@ -102,6 +102,23 @@ public class TestReplacePartitions extends TestBase {
.withRecordCount(1)
.build();
+ static final PartitionSpec SPEC_ALL_VOID =
+ PartitionSpec.builderFor(SCHEMA).alwaysNull("id").alwaysNull("data").build();
+
+ static final DataFile FILE_ALL_VOID_UNPARTITIONED_A =
+ DataFiles.builder(SPEC_ALL_VOID)
+ .withPath("/path/to/data-all-void-unpartitioned-a.parquet")
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build();
+
+ static final DataFile FILE_ALL_VOID_UNPARTITIONED_B =
+ DataFiles.builder(SPEC_ALL_VOID)
+ .withPath("/path/to/data-all-void-unpartitioned-b.parquet")
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build();
+
@Parameter(index = 1)
private String branch;
@@ -198,6 +215,42 @@ public class TestReplacePartitions extends TestBase {
statuses(Status.DELETED));
}
+ @TestTemplate
+ public void testReplaceAllVoidUnpartitionedTable() {
+ Table tableVoid =
+ TestTables.create(tableDir, "allvoidUnpartitioned", SCHEMA, SPEC_ALL_VOID, formatVersion);
+
+ commit(tableVoid, tableVoid.newAppend().appendFile(FILE_ALL_VOID_UNPARTITIONED_A), branch);
+ validateSnapshot(
+ null,
+ latestSnapshot(TestTables.readMetadata("allvoidUnpartitioned"), branch),
+ FILE_ALL_VOID_UNPARTITIONED_A);
+
+ ReplacePartitions replacePartitions =
+ tableVoid.newReplacePartitions().addFile(FILE_ALL_VOID_UNPARTITIONED_B);
+ commit(tableVoid, replacePartitions, branch);
+
+ assertThat(TestTables.metadataVersion("allvoidUnpartitioned")).isEqualTo(2);
+ TableMetadata replaceMetadata = TestTables.readMetadata("allvoidUnpartitioned");
+ long replaceId = latestSnapshot(replaceMetadata, branch).snapshotId();
+ List<ManifestFile> manifestFiles =
+ latestSnapshot(replaceMetadata, branch).allManifests(tableVoid.io());
+
+ assertThat(manifestFiles).hasSize(2);
+
+ validateManifestEntries(
+ manifestFiles.get(0),
+ ids(replaceId),
+ files(FILE_ALL_VOID_UNPARTITIONED_B),
+ statuses(Status.ADDED));
+
+ validateManifestEntries(
+ manifestFiles.get(1),
+ ids(replaceId),
+ files(FILE_ALL_VOID_UNPARTITIONED_A),
+ statuses(Status.DELETED));
+ }
+
@TestTemplate
public void testReplaceAndMergeWithUnpartitionedTable() throws IOException {
Table unpartitioned =
(iceberg) 02/03: AWS: exclude logging classes from bundle (#14225)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit cc51275492910beca9cecd45209f464bfb039b70
Author: Daniel Weeks <dw...@apache.org>
AuthorDate: Tue Sep 30 16:03:14 2025 -0700
AWS: exclude logging classes from bundle (#14225)
(cherry picked from commit 5c6629ef915061967fa67e4b981b0151cf8d42d4)
---
aws-bundle/build.gradle | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/aws-bundle/build.gradle b/aws-bundle/build.gradle
index 9e937aa0ed..5b9054812a 100644
--- a/aws-bundle/build.gradle
+++ b/aws-bundle/build.gradle
@@ -53,7 +53,9 @@ project(":iceberg-aws-bundle") {
}
dependencies {
- exclude(dependency('org.slf4j:slf4j-api'))
+ exclude(dependency('org.slf4j:.*'))
+ exclude(dependency('org.apache.logging.log4j:.*'))
+ exclude(dependency('org.apache.logging.slf4j:.*'))
}
// relocate AWS-specific versions