You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2026/10/02 13:19:19 UTC
(asterixdb) 01/02: [NO ISSUE][EXT] Advance iceberg to 1.12.0
This is an automated email from the ASF dual-hosted git repository.
parshimers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0957a329542fcd3034b01c9e42c77a66e28877d1
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Oct 2 07:07:42 2026 -0400
[NO ISSUE][EXT] Advance iceberg to 1.12.0
- user model changes: no
- storage format changes: no
- interface changes: no
Details: Move iceberg from 1.11.0 to 1.12.0, and nessie with it from
0.107.5 to 0.108.8 (the version iceberg-nessie 1.12.0 declares). The
forked org.apache.iceberg.gcp.gcs.PrefixedStorage is identical upstream
at both tags, so iceberg-gcs-fork.version moves with no re-copy.
1.12.0 removes GenericAppenderFactory. The two tests that wrote Iceberg
data files with it now use FormatModelRegistry.dataWriteBuilder, whose
DataWriter yields the DataFile directly.
Ext-ref: MB-72431
Generated-by: Claude Opus 5.5 <no...@anthropic.com>
Change-Id: If2255f3d9bf6d45ef19f16b451c889d8abcff3dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21971
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
---
.../aws/AwsS3ExternalDatasetTest.java | 20 ++++++++++----------
.../asterix/test/runtime/SqlppHdfsExecutionTest.java | 20 ++++++++++----------
asterixdb/pom.xml | 6 +++---
3 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index d341800db8..300db67024 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -62,19 +62,18 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.iceberg.DataFile;
-import org.apache.iceberg.DataFiles;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.Tables;
-import org.apache.iceberg.data.GenericAppenderFactory;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
-import org.apache.iceberg.hadoop.HadoopInputFile;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.formats.FormatModelRegistry;
import org.apache.iceberg.hadoop.HadoopTables;
-import org.apache.iceberg.io.FileAppender;
+import org.apache.iceberg.io.DataWriter;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -227,14 +226,15 @@ public class AwsS3ExternalDatasetTest {
FileFormat fileFormat = FileFormat.fromFileName(filename);
Preconditions.checkNotNull(fileFormat, "Cannot determine format for file: %s", filename);
- FileAppender<Record> fileAppender = new GenericAppenderFactory(AwsS3ExternalDatasetTest.SCHEMA)
- .newAppender(fromPath(path, CONF), fileFormat);
- try (FileAppender<Record> appender = fileAppender) {
- appender.addAll(records);
+ DataWriter<Record> writer = FormatModelRegistry
+ .<Record, Object> dataWriteBuilder(fileFormat, Record.class,
+ EncryptedFiles.plainAsEncryptedOutput(fromPath(path, CONF)))
+ .schema(AwsS3ExternalDatasetTest.SCHEMA).spec(PartitionSpec.unpartitioned()).build();
+ try (DataWriter<Record> w = writer) {
+ w.write(records);
}
- return DataFiles.builder(PartitionSpec.unpartitioned()).withInputFile(HadoopInputFile.fromPath(path, CONF))
- .withMetrics(fileAppender.metrics()).build();
+ return writer.toDataFile();
}
private static void prepareIcebergConfiguration() {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java
index 412fbc1af6..d3b04c103d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java
@@ -50,19 +50,18 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.iceberg.DataFile;
-import org.apache.iceberg.DataFiles;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.Tables;
-import org.apache.iceberg.data.GenericAppenderFactory;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
-import org.apache.iceberg.hadoop.HadoopInputFile;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.formats.FormatModelRegistry;
import org.apache.iceberg.hadoop.HadoopTables;
-import org.apache.iceberg.io.FileAppender;
+import org.apache.iceberg.io.DataWriter;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -98,14 +97,15 @@ public class SqlppHdfsExecutionTest {
FileFormat fileFormat = FileFormat.fromFileName(filename);
Preconditions.checkNotNull(fileFormat, "Cannot determine format for file: %s", filename);
- FileAppender<Record> fileAppender =
- new GenericAppenderFactory(schema).newAppender(fromPath(path, CONF), fileFormat);
- try (FileAppender<Record> appender = fileAppender) {
- appender.addAll(records);
+ DataWriter<Record> writer = FormatModelRegistry
+ .<Record, Object> dataWriteBuilder(fileFormat, Record.class,
+ EncryptedFiles.plainAsEncryptedOutput(fromPath(path, CONF)))
+ .schema(schema).spec(PartitionSpec.unpartitioned()).build();
+ try (DataWriter<Record> w = writer) {
+ w.write(records);
}
- return DataFiles.builder(PartitionSpec.unpartitioned()).withInputFile(HadoopInputFile.fromPath(path, CONF))
- .withMetrics(fileAppender.metrics()).build();
+ return writer.toDataFile();
}
private static void setUpIcebergData() {
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 4a30c500b9..f9371bc0b3 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -123,7 +123,7 @@
<io.opencensus.version>0.31.1</io.opencensus.version>
<protobuf-java.version>4.36.2</protobuf-java.version>
- <icebergjavasdk.version>1.11.0</icebergjavasdk.version>
+ <icebergjavasdk.version>1.12.0</icebergjavasdk.version>
<!-- The org.apache.iceberg.gcp.gcs package is forked into asterix-external-data,
under util/google/iceberg/fileio. This records the release those files were
@@ -134,10 +134,10 @@
The marker below is a one-line comment adjacent to the property on purpose:
the updater looks only one line back for it. -->
<!-- @pinned-with: ${icebergjavasdk.version} -->
- <iceberg-gcs-fork.version>1.11.0</iceberg-gcs-fork.version>
+ <iceberg-gcs-fork.version>1.12.0</iceberg-gcs-fork.version>
<!-- @derived-from: org.apache.iceberg:iceberg-nessie:${icebergjavasdk.version}#dep:nessie-client -->
- <nessieproject.version>0.107.5</nessieproject.version>
+ <nessieproject.version>0.108.8</nessieproject.version>
<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>