You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2024/04/16 15:38:03 UTC
(iceberg) 02/03: Flink: Recover flink/1.18 files from history
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit f761d98a1d53cdb6574aeb84733158924635f25f
Author: Rodrigo Meneses <rm...@apple.com>
AuthorDate: Tue Apr 16 07:44:22 2024 -0700
Flink: Recover flink/1.18 files from history
---
flink/v1.18/build.gradle | 266 +++++
flink/v1.18/flink-runtime/LICENSE | 502 +++++++++
flink/v1.18/flink-runtime/NOTICE | 91 ++
.../iceberg/flink/IcebergConnectorSmokeTest.java | 29 +
.../sink/shuffle/MapRangePartitionerBenchmark.java | 199 ++++
.../org/apache/iceberg/flink/CatalogLoader.java | 215 ++++
.../org/apache/iceberg/flink/FlinkCatalog.java | 833 ++++++++++++++
.../apache/iceberg/flink/FlinkCatalogFactory.java | 213 ++++
.../org/apache/iceberg/flink/FlinkConfParser.java | 261 +++++
.../apache/iceberg/flink/FlinkConfigOptions.java | 107 ++
.../iceberg/flink/FlinkDynamicTableFactory.java | 208 ++++
.../iceberg/flink/FlinkEnvironmentContext.java | 31 +
.../org/apache/iceberg/flink/FlinkFilters.java | 266 +++++
.../org/apache/iceberg/flink/FlinkFixupTypes.java | 50 +
.../org/apache/iceberg/flink/FlinkReadConf.java | 213 ++++
.../org/apache/iceberg/flink/FlinkReadOptions.java | 123 ++
.../org/apache/iceberg/flink/FlinkSchemaUtil.java | 232 ++++
.../apache/iceberg/flink/FlinkSourceFilter.java | 49 +
.../org/apache/iceberg/flink/FlinkTypeToType.java | 203 ++++
.../org/apache/iceberg/flink/FlinkTypeVisitor.java | 80 ++
.../org/apache/iceberg/flink/FlinkWriteConf.java | 205 ++++
.../apache/iceberg/flink/FlinkWriteOptions.java | 73 ++
.../org/apache/iceberg/flink/IcebergTableSink.java | 121 ++
.../org/apache/iceberg/flink/RowDataWrapper.java | 142 +++
.../java/org/apache/iceberg/flink/TableLoader.java | 159 +++
.../org/apache/iceberg/flink/TypeToFlinkType.java | 134 +++
.../org/apache/iceberg/flink/actions/Actions.java | 52 +
.../flink/actions/RewriteDataFilesAction.java | 72 ++
.../flink/data/AvroWithFlinkSchemaVisitor.java | 75 ++
.../apache/iceberg/flink/data/FlinkAvroReader.java | 169 +++
.../apache/iceberg/flink/data/FlinkAvroWriter.java | 165 +++
.../apache/iceberg/flink/data/FlinkOrcReader.java | 131 +++
.../apache/iceberg/flink/data/FlinkOrcReaders.java | 283 +++++
.../apache/iceberg/flink/data/FlinkOrcWriter.java | 163 +++
.../apache/iceberg/flink/data/FlinkOrcWriters.java | 317 ++++++
.../iceberg/flink/data/FlinkParquetReaders.java | 832 ++++++++++++++
.../iceberg/flink/data/FlinkParquetWriters.java | 504 +++++++++
.../iceberg/flink/data/FlinkSchemaVisitor.java | 161 +++
.../iceberg/flink/data/FlinkValueReaders.java | 312 ++++++
.../iceberg/flink/data/FlinkValueWriters.java | 253 +++++
.../flink/data/ParquetWithFlinkSchemaVisitor.java | 222 ++++
.../iceberg/flink/data/RowDataProjection.java | 341 ++++++
.../org/apache/iceberg/flink/data/RowDataUtil.java | 100 ++
.../apache/iceberg/flink/data/StructRowData.java | 300 +++++
.../sink/AvroGenericRecordToRowDataMapper.java | 61 +
.../iceberg/flink/sink/BaseDeltaTaskWriter.java | 125 +++
.../flink/sink/BucketPartitionKeySelector.java | 70 ++
.../iceberg/flink/sink/BucketPartitioner.java | 103 ++
.../iceberg/flink/sink/BucketPartitionerUtil.java | 125 +++
.../iceberg/flink/sink/CachingTableSupplier.java | 91 ++
.../apache/iceberg/flink/sink/CommitSummary.java | 93 ++
.../apache/iceberg/flink/sink/DeltaManifests.java | 71 ++
.../flink/sink/DeltaManifestsSerializer.java | 122 ++
.../flink/sink/EqualityFieldKeySelector.java | 86 ++
.../iceberg/flink/sink/FlinkAppenderFactory.java | 274 +++++
.../iceberg/flink/sink/FlinkFileWriterFactory.java | 293 +++++
.../iceberg/flink/sink/FlinkManifestUtil.java | 132 +++
.../org/apache/iceberg/flink/sink/FlinkSink.java | 654 +++++++++++
.../iceberg/flink/sink/IcebergFilesCommitter.java | 516 +++++++++
.../flink/sink/IcebergFilesCommitterMetrics.java | 73 ++
.../iceberg/flink/sink/IcebergStreamWriter.java | 120 ++
.../flink/sink/IcebergStreamWriterMetrics.java | 89 ++
.../flink/sink/ManifestOutputFileFactory.java | 94 ++
.../iceberg/flink/sink/PartitionKeySelector.java | 64 ++
.../iceberg/flink/sink/PartitionedDeltaWriter.java | 97 ++
.../flink/sink/RowDataTaskWriterFactory.java | 244 ++++
.../iceberg/flink/sink/TaskWriterFactory.java | 45 +
.../flink/sink/UnpartitionedDeltaWriter.java | 69 ++
.../flink/sink/shuffle/AggregatedStatistics.java | 71 ++
.../sink/shuffle/AggregatedStatisticsTracker.java | 133 +++
.../iceberg/flink/sink/shuffle/DataStatistics.java | 57 +
.../sink/shuffle/DataStatisticsCoordinator.java | 396 +++++++
.../shuffle/DataStatisticsCoordinatorProvider.java | 51 +
.../flink/sink/shuffle/DataStatisticsEvent.java | 57 +
.../flink/sink/shuffle/DataStatisticsOperator.java | 197 ++++
.../flink/sink/shuffle/DataStatisticsOrRecord.java | 113 ++
.../shuffle/DataStatisticsOrRecordSerializer.java | 219 ++++
.../flink/sink/shuffle/DataStatisticsUtil.java | 96 ++
.../flink/sink/shuffle/MapDataStatistics.java | 70 ++
.../sink/shuffle/MapDataStatisticsSerializer.java | 187 ++++
.../flink/sink/shuffle/MapRangePartitioner.java | 381 +++++++
.../flink/sink/shuffle/SortKeySerializer.java | 353 ++++++
.../AvroGenericRecordFileScanTaskReader.java | 42 +
.../apache/iceberg/flink/source/DataIterator.java | 156 +++
.../iceberg/flink/source/DataTaskReader.java | 47 +
.../iceberg/flink/source/FileScanTaskReader.java | 35 +
.../iceberg/flink/source/FlinkInputFormat.java | 141 +++
.../iceberg/flink/source/FlinkInputSplit.java | 48 +
.../apache/iceberg/flink/source/FlinkSource.java | 310 ++++++
.../iceberg/flink/source/FlinkSplitPlanner.java | 189 ++++
.../apache/iceberg/flink/source/IcebergSource.java | 543 +++++++++
.../iceberg/flink/source/IcebergTableSource.java | 229 ++++
.../flink/source/RowDataFileScanTaskReader.java | 243 ++++
.../iceberg/flink/source/RowDataRewriter.java | 172 +++
.../RowDataToAvroGenericRecordConverter.java | 70 ++
.../apache/iceberg/flink/source/ScanContext.java | 592 ++++++++++
.../apache/iceberg/flink/source/SourceUtil.java | 77 ++
.../flink/source/StreamingMonitorFunction.java | 269 +++++
.../flink/source/StreamingReaderOperator.java | 246 ++++
.../flink/source/StreamingStartingStrategy.java | 54 +
.../source/assigner/DefaultSplitAssigner.java | 119 ++
.../flink/source/assigner/GetSplitResult.java | 77 ++
.../assigner/OrderedSplitAssignerFactory.java | 46 +
.../assigner/SimpleSplitAssignerFactory.java | 37 +
.../flink/source/assigner/SplitAssigner.java | 124 +++
.../source/assigner/SplitAssignerFactory.java | 30 +
.../flink/source/assigner/SplitAssignerType.java | 33 +
.../enumerator/AbstractIcebergEnumerator.java | 175 +++
.../enumerator/ContinuousEnumerationResult.java | 57 +
.../enumerator/ContinuousIcebergEnumerator.java | 187 ++++
.../source/enumerator/ContinuousSplitPlanner.java | 30 +
.../enumerator/ContinuousSplitPlannerImpl.java | 240 ++++
.../source/enumerator/EnumerationHistory.java | 96 ++
.../enumerator/IcebergEnumeratorPosition.java | 79 ++
.../IcebergEnumeratorPositionSerializer.java | 90 ++
.../source/enumerator/IcebergEnumeratorState.java | 65 ++
.../IcebergEnumeratorStateSerializer.java | 196 ++++
.../source/enumerator/StaticIcebergEnumerator.java | 51 +
.../flink/source/reader/ArrayBatchRecords.java | 171 +++
.../reader/ArrayPoolDataIteratorBatcher.java | 130 +++
.../reader/AvroGenericRecordReaderFunction.java | 102 ++
.../reader/ColumnStatsWatermarkExtractor.java | 98 ++
.../flink/source/reader/DataIteratorBatcher.java | 36 +
.../source/reader/DataIteratorReaderFunction.java | 43 +
.../flink/source/reader/IcebergSourceReader.java | 77 ++
.../source/reader/IcebergSourceReaderMetrics.java | 61 +
.../source/reader/IcebergSourceSplitReader.java | 167 +++
.../flink/source/reader/ListBatchRecords.java | 85 ++
.../source/reader/ListDataIteratorBatcher.java | 94 ++
.../source/reader/MetaDataReaderFunction.java | 65 ++
.../flink/source/reader/ReaderFunction.java | 31 +
.../flink/source/reader/RecordAndPosition.java | 78 ++
.../iceberg/flink/source/reader/RecordFactory.java | 34 +
.../flink/source/reader/RowDataReaderFunction.java | 79 ++
.../flink/source/reader/RowDataRecordFactory.java | 61 +
.../source/reader/SerializableRecordEmitter.java | 40 +
.../source/reader/SplitWatermarkExtractor.java | 28 +
.../reader/WatermarkExtractorRecordEmitter.java | 67 ++
.../flink/source/split/IcebergSourceSplit.java | 177 +++
.../source/split/IcebergSourceSplitSerializer.java | 60 +
.../source/split/IcebergSourceSplitState.java | 37 +
.../source/split/IcebergSourceSplitStatus.java | 25 +
.../flink/source/split/SerializableComparator.java | 24 +
.../flink/source/split/SplitComparators.java | 76 ++
.../flink/source/split/SplitRequestEvent.java | 54 +
.../iceberg/flink/util/ElapsedTimeGauge.java | 47 +
.../iceberg/flink/util/FlinkAlterTableUtil.java | 248 +++++
.../iceberg/flink/util/FlinkCompatibilityUtil.java | 47 +
.../apache/iceberg/flink/util/FlinkPackage.java | 61 +
.../org.apache.flink.table.factories.Factory | 16 +
.../org.apache.flink.table.factories.TableFactory | 16 +
.../flink/AvroGenericRecordConverterBase.java | 90 ++
.../org/apache/iceberg/flink/CatalogTestBase.java | 143 +++
.../org/apache/iceberg/flink/DataGenerator.java | 42 +
.../org/apache/iceberg/flink/DataGenerators.java | 1172 ++++++++++++++++++++
.../org/apache/iceberg/flink/FlinkTestBase.java | 129 +++
.../iceberg/flink/HadoopCatalogExtension.java | 104 ++
.../iceberg/flink/HadoopCatalogResource.java | 90 ++
.../apache/iceberg/flink/HadoopTableResource.java | 64 ++
.../apache/iceberg/flink/MiniClusterResource.java | 68 ++
.../iceberg/flink/MiniFlinkClusterExtension.java | 53 +
.../org/apache/iceberg/flink/RowDataConverter.java | 135 +++
.../org/apache/iceberg/flink/SimpleDataUtil.java | 443 ++++++++
.../java/org/apache/iceberg/flink/TestBase.java | 130 +++
.../apache/iceberg/flink/TestCatalogLoader.java | 118 ++
.../iceberg/flink/TestCatalogTableLoader.java | 116 ++
.../apache/iceberg/flink/TestChangeLogTable.java | 305 +++++
.../iceberg/flink/TestDataFileSerialization.java | 203 ++++
.../org/apache/iceberg/flink/TestFixtures.java | 59 +
.../iceberg/flink/TestFlinkAnonymousTable.java | 64 ++
.../iceberg/flink/TestFlinkCatalogDatabase.java | 254 +++++
.../iceberg/flink/TestFlinkCatalogFactory.java | 119 ++
.../iceberg/flink/TestFlinkCatalogTable.java | 672 +++++++++++
.../flink/TestFlinkCatalogTablePartitions.java | 119 ++
.../apache/iceberg/flink/TestFlinkConfParser.java | 61 +
.../org/apache/iceberg/flink/TestFlinkFilters.java | 462 ++++++++
.../apache/iceberg/flink/TestFlinkHiveCatalog.java | 103 ++
.../apache/iceberg/flink/TestFlinkSchemaUtil.java | 416 +++++++
.../apache/iceberg/flink/TestFlinkTableSink.java | 358 ++++++
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 334 ++++++
.../java/org/apache/iceberg/flink/TestHelpers.java | 628 +++++++++++
.../apache/iceberg/flink/TestIcebergConnector.java | 350 ++++++
.../flink/TestManifestFileSerialization.java | 173 +++
.../apache/iceberg/flink/TestRowDataWrapper.java | 93 ++
.../org/apache/iceberg/flink/TestTableLoader.java | 57 +
.../iceberg/flink/TestTableSerialization.java | 110 ++
.../flink/actions/TestRewriteDataFilesAction.java | 481 ++++++++
.../apache/iceberg/flink/data/RandomRowData.java | 38 +
.../iceberg/flink/data/RowDataToRowMapper.java | 50 +
.../flink/data/TestFlinkAvroReaderWriter.java | 185 +++
.../flink/data/TestFlinkOrcReaderWriter.java | 107 ++
.../iceberg/flink/data/TestFlinkParquetReader.java | 138 +++
.../iceberg/flink/data/TestFlinkParquetWriter.java | 94 ++
.../iceberg/flink/data/TestRowDataProjection.java | 593 ++++++++++
.../iceberg/flink/data/TestRowProjection.java | 596 ++++++++++
.../iceberg/flink/data/TestStructRowData.java | 100 ++
.../sink/TestAvroGenericRecordToRowDataMapper.java | 38 +
.../flink/sink/TestBucketPartitionKeySelector.java | 65 ++
.../iceberg/flink/sink/TestBucketPartitioner.java | 107 ++
.../TestBucketPartitionerFlinkIcebergSink.java | 227 ++++
.../flink/sink/TestBucketPartitionerUtil.java | 126 +++
.../flink/sink/TestCachingTableSupplier.java | 81 ++
.../flink/sink/TestCompressionSettings.java | 254 +++++
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 429 +++++++
.../flink/sink/TestFlinkAppenderFactory.java | 65 ++
.../flink/sink/TestFlinkFileWriterFactory.java | 66 ++
.../iceberg/flink/sink/TestFlinkIcebergSink.java | 397 +++++++
.../flink/sink/TestFlinkIcebergSinkBase.java | 64 ++
.../flink/sink/TestFlinkIcebergSinkBranch.java | 137 +++
.../iceberg/flink/sink/TestFlinkIcebergSinkV2.java | 236 ++++
.../flink/sink/TestFlinkIcebergSinkV2Base.java | 362 ++++++
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 134 +++
.../iceberg/flink/sink/TestFlinkManifest.java | 315 ++++++
.../flink/sink/TestFlinkPartitioningWriters.java | 77 ++
.../flink/sink/TestFlinkPositionDeltaWriters.java | 66 ++
.../flink/sink/TestFlinkRollingFileWriters.java | 51 +
.../iceberg/flink/sink/TestFlinkWriterMetrics.java | 60 +
.../flink/sink/TestIcebergFilesCommitter.java | 1148 +++++++++++++++++++
.../flink/sink/TestIcebergStreamWriter.java | 388 +++++++
.../flink/sink/TestRowDataPartitionKey.java | 252 +++++
.../apache/iceberg/flink/sink/TestTaskWriters.java | 243 ++++
.../sink/shuffle/TestAggregatedStatistics.java | 63 ++
.../shuffle/TestAggregatedStatisticsTracker.java | 178 +++
.../shuffle/TestDataStatisticsCoordinator.java | 178 +++
.../TestDataStatisticsCoordinatorProvider.java | 143 +++
.../sink/shuffle/TestDataStatisticsOperator.java | 255 +++++
.../flink/sink/shuffle/TestMapDataStatistics.java | 90 ++
.../sink/shuffle/TestMapRangePartitioner.java | 448 ++++++++
.../sink/shuffle/TestSortKeySerializerBase.java | 65 ++
.../shuffle/TestSortKeySerializerNestedStruct.java | 55 +
.../shuffle/TestSortKeySerializerPrimitives.java | 57 +
.../iceberg/flink/source/BoundedTableFactory.java | 170 +++
.../iceberg/flink/source/BoundedTestSource.java | 108 ++
.../flink/source/ChangeLogTableTestBase.java | 87 ++
.../apache/iceberg/flink/source/SplitHelpers.java | 132 +++
.../apache/iceberg/flink/source/SqlHelpers.java | 60 +
.../flink/source/TestBoundedTableFactory.java | 87 ++
.../iceberg/flink/source/TestFlinkInputFormat.java | 191 ++++
.../source/TestFlinkInputFormatReaderDeletes.java | 71 ++
.../flink/source/TestFlinkMergingMetrics.java | 71 ++
.../flink/source/TestFlinkMetaDataTable.java | 813 ++++++++++++++
.../flink/source/TestFlinkReaderDeletesBase.java | 90 ++
.../apache/iceberg/flink/source/TestFlinkScan.java | 540 +++++++++
.../iceberg/flink/source/TestFlinkScanSql.java | 69 ++
.../iceberg/flink/source/TestFlinkSource.java | 90 ++
.../flink/source/TestFlinkSourceConfig.java | 58 +
.../iceberg/flink/source/TestFlinkSourceSql.java | 85 ++
.../iceberg/flink/source/TestFlinkTableSource.java | 614 ++++++++++
.../flink/source/TestIcebergSourceBounded.java | 127 +++
.../TestIcebergSourceBoundedGenericRecord.java | 203 ++++
.../flink/source/TestIcebergSourceBoundedSql.java | 76 ++
.../flink/source/TestIcebergSourceContinuous.java | 543 +++++++++
.../flink/source/TestIcebergSourceFailover.java | 297 +++++
...cebergSourceFailoverWithWatermarkExtractor.java | 112 ++
.../source/TestIcebergSourceReaderDeletes.java | 102 ++
.../iceberg/flink/source/TestIcebergSourceSql.java | 158 +++
.../TestIcebergSourceWithWatermarkExtractor.java | 408 +++++++
.../source/TestMetadataTableReadableMetrics.java | 299 +++++
.../flink/source/TestProjectMetaColumn.java | 186 ++++
.../TestRowDataToAvroGenericRecordConverter.java | 36 +
.../iceberg/flink/source/TestSourceUtil.java | 60 +
.../apache/iceberg/flink/source/TestSqlBase.java | 158 +++
.../iceberg/flink/source/TestStreamScanSql.java | 430 +++++++
.../flink/source/TestStreamingMonitorFunction.java | 402 +++++++
.../flink/source/TestStreamingReaderOperator.java | 293 +++++
.../source/assigner/SplitAssignerTestBase.java | 130 +++
.../source/assigner/TestDefaultSplitAssigner.java | 43 +
.../TestFileSequenceNumberBasedSplitAssigner.java | 80 ++
.../assigner/TestWatermarkBasedSplitAssigner.java | 146 +++
.../enumerator/ManualContinuousSplitPlanner.java | 97 ++
.../TestContinuousIcebergEnumerator.java | 352 ++++++
.../enumerator/TestContinuousSplitPlannerImpl.java | 692 ++++++++++++
...estContinuousSplitPlannerImplStartStrategy.java | 205 ++++
.../source/enumerator/TestEnumerationHistory.java | 134 +++
.../TestIcebergEnumeratorStateSerializer.java | 145 +++
.../source/reader/ReaderFunctionTestBase.java | 219 ++++
.../iceberg/flink/source/reader/ReaderUtil.java | 125 +++
.../flink/source/reader/TestArrayBatchRecords.java | 68 ++
.../TestArrayPoolDataIteratorBatcherRowData.java | 358 ++++++
.../reader/TestColumnStatsWatermarkExtractor.java | 178 +++
.../source/reader/TestIcebergSourceReader.java | 184 +++
.../source/reader/TestRowDataReaderFunction.java | 74 ++
.../flink/source/reader/TestingMetricGroup.java | 102 ++
.../split/TestIcebergSourceSplitSerializer.java | 162 +++
.../iceberg/flink/util/TestFlinkPackage.java | 55 +
.../org.apache.flink.table.factories.Factory | 16 +
286 files changed, 51109 insertions(+)
diff --git a/flink/v1.18/build.gradle b/flink/v1.18/build.gradle
new file mode 100644
index 0000000000..c08ae5d8cc
--- /dev/null
+++ b/flink/v1.18/build.gradle
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+String flinkMajorVersion = '1.18'
+String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
+
+project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
+
+ dependencies {
+ implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+ api project(':iceberg-api')
+ implementation project(':iceberg-common')
+ implementation project(':iceberg-core')
+ api project(':iceberg-data')
+ implementation project(':iceberg-orc')
+ implementation project(':iceberg-parquet')
+ implementation project(':iceberg-hive-metastore')
+
+ compileOnly libs.flink118.avro
+ // for dropwizard histogram metrics implementation
+ compileOnly libs.flink118.metrics.dropwizard
+ compileOnly libs.flink118.streaming.java
+ compileOnly "${libs.flink118.streaming.java.get().module}:${libs.flink118.streaming.java.get().getVersion()}:tests"
+ compileOnly libs.flink118.table.api.java.bridge
+ compileOnly "org.apache.flink:flink-table-planner_${scalaVersion}:${libs.versions.flink118.get()}"
+ compileOnly libs.flink118.connector.base
+ compileOnly libs.flink118.connector.files
+
+ compileOnly libs.hadoop2.hdfs
+ compileOnly libs.hadoop2.common
+ compileOnly(libs.hadoop2.minicluster) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ }
+
+ implementation(libs.parquet.avro) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ // already shaded by Parquet
+ exclude group: 'it.unimi.dsi'
+ exclude group: 'org.codehaus.jackson'
+ }
+
+ compileOnly libs.avro.avro
+
+ implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
+ exclude group: 'org.apache.hadoop'
+ exclude group: 'commons-lang'
+ // These artifacts are shaded and included in the orc-core fat jar
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.hive', module: 'hive-storage-api'
+ exclude group: 'org.slf4j'
+ }
+
+ testImplementation libs.flink118.connector.test.utils
+ testImplementation libs.flink118.core
+ testImplementation libs.flink118.runtime
+ testImplementation(libs.flink118.test.utilsjunit) {
+ exclude group: 'junit'
+ }
+ testImplementation(libs.flink118.test.utils) {
+ exclude group: "org.apache.curator", module: 'curator-test'
+ exclude group: 'junit'
+ }
+
+ testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
+
+ // By default, hive-exec is a fat/uber jar and it exports a guava library
+ // that's really old. We use the core classifier to be able to override our guava
+ // version. Luckily, hive-exec seems to work okay so far with this version of guava
+ // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
+ testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hive', module: 'hive-llap-tez'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.calcite'
+ exclude group: 'org.apache.calcite.avatica'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+
+ testImplementation(libs.hive2.metastore) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hbase'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'co.cask.tephra'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+ exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+ exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+ exclude group: 'com.tdunning', module: 'json'
+ exclude group: 'javax.transaction', module: 'transaction-api'
+ exclude group: 'com.zaxxer', module: 'HikariCP'
+ exclude group: 'org.slf4j'
+ }
+
+ testImplementation libs.awaitility
+ testImplementation libs.assertj.core
+ }
+
+ test {
+ useJUnitPlatform()
+ }
+}
+
+project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
+ apply plugin: 'com.github.johnrengelman.shadow'
+
+ tasks.jar.dependsOn tasks.shadowJar
+
+ sourceSets {
+ integration {
+ java.srcDir "$projectDir/src/integration/java"
+ resources.srcDir "$projectDir/src/integration/resources"
+ }
+ }
+
+ configurations {
+ implementation {
+ // included in Flink
+ exclude group: 'org.slf4j'
+ exclude group: 'org.apache.commons'
+ exclude group: 'commons-pool'
+ exclude group: 'commons-codec'
+ exclude group: 'org.xerial.snappy'
+ exclude group: 'javax.xml.bind'
+ exclude group: 'javax.annotation'
+ }
+ }
+
+ dependencies {
+ implementation(project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}")) {
+ exclude group: 'org.apache.flink'
+ }
+ implementation project(':iceberg-aws')
+ implementation project(':iceberg-azure')
+ implementation(project(':iceberg-aliyun')) {
+ exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
+ exclude group: 'org.apache.httpcomponents', module: 'httpclient'
+ exclude group: 'commons-logging', module: 'commons-logging'
+ }
+ implementation project(':iceberg-gcp')
+ implementation(project(':iceberg-nessie')) {
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+
+ // for dropwizard histogram metrics implementation
+ implementation libs.flink118.metrics.dropwizard
+
+ // for integration testing with the flink-runtime-jar
+ // all of those dependencies are required because the integration test extends FlinkTestBase
+ integrationCompileOnly project(':iceberg-api')
+ integrationImplementation libs.junit.vintage.engine
+ integrationImplementation libs.assertj.core
+ integrationImplementation project(path: ":iceberg-flink:iceberg-flink-${flinkMajorVersion}", configuration: "testArtifacts")
+ integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ integrationImplementation(libs.flink118.test.utils) {
+ exclude group: "org.apache.curator", module: 'curator-test'
+ exclude group: 'junit'
+ }
+
+ integrationImplementation libs.flink118.table.api.java.bridge
+ integrationImplementation "org.apache.flink:flink-table-planner_${scalaVersion}:${libs.versions.flink118.get()}"
+
+ integrationImplementation libs.hadoop2.common
+ integrationImplementation libs.hadoop2.hdfs
+ integrationImplementation(libs.hadoop2.minicluster) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ }
+
+ integrationImplementation(libs.hive2.metastore) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hbase'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'co.cask.tephra'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+ exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+ exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+ exclude group: 'com.tdunning', module: 'json'
+ exclude group: 'javax.transaction', module: 'transaction-api'
+ exclude group: 'com.zaxxer', module: 'HikariCP'
+ exclude group: 'org.slf4j'
+ }
+
+ integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hive', module: 'hive-llap-tez'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.calcite'
+ exclude group: 'org.apache.calcite.avatica'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+ }
+
+ shadowJar {
+ configurations = [project.configurations.runtimeClasspath]
+
+ zip64 true
+
+ // include the LICENSE and NOTICE files for the shaded Jar
+ from(projectDir) {
+ include 'LICENSE'
+ include 'NOTICE'
+ }
+
+ // Relocate dependencies to avoid conflicts
+ relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
+ relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
+ relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
+ relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
+ relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
+ relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
+ relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
+ relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
+ relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
+ relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
+ relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
+ relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
+ relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
+
+ archiveClassifier.set(null)
+ }
+
+ task integrationTest(type: Test) {
+ description = "Test Flink Runtime Jar against Flink ${flinkMajorVersion}"
+ group = "verification"
+ jvmArgs += project.property('extraJvmArgs')
+ testClassesDirs = sourceSets.integration.output.classesDirs
+ classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
+ inputs.file(shadowJar.archiveFile.get().asFile.path)
+ }
+ integrationTest.dependsOn shadowJar
+ check.dependsOn integrationTest
+
+ jar {
+ enabled = false
+ }
+}
diff --git a/flink/v1.18/flink-runtime/LICENSE b/flink/v1.18/flink-runtime/LICENSE
new file mode 100644
index 0000000000..8ab53469eb
--- /dev/null
+++ b/flink/v1.18/flink-runtime/LICENSE
@@ -0,0 +1,502 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache Avro.
+
+Copyright: 2014-2020 The Apache Software Foundation.
+Home page: https://parquet.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains the Jackson JSON processor.
+
+Copyright: 2007-2020 Tatu Saloranta and other contributors
+Home page: http://jackson.codehaus.org/
+License: http://www.apache.org/licenses/LICENSE-2.0.txt
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache Parquet.
+
+Copyright: 2014-2020 The Apache Software Foundation.
+Home page: https://parquet.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache Thrift.
+
+Copyright: 2006-2010 The Apache Software Foundation.
+Home page: https://thrift.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains fastutil.
+
+Copyright: 2002-2014 Sebastiano Vigna
+Home page: http://fastutil.di.unimi.it/
+License: http://www.apache.org/licenses/LICENSE-2.0.html
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache ORC.
+
+Copyright: 2013-2020 The Apache Software Foundation.
+Home page: https://orc.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache Hive's storage API via ORC.
+
+Copyright: 2013-2020 The Apache Software Foundation.
+Home page: https://hive.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Airlift Aircompressor.
+
+Copyright: 2011-2020 Aircompressor authors.
+Home page: https://github.com/airlift/aircompressor
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Airlift Slice.
+
+Copyright: 2013-2020 Slice authors.
+Home page: https://github.com/airlift/slice
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains JetBrains annotations.
+
+Copyright: 2000-2020 JetBrains s.r.o.
+Home page: https://github.com/JetBrains/java-annotations
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Google Guava.
+
+Copyright: 2006-2020 The Guava Authors
+Home page: https://github.com/google/guava
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Google Error Prone Annotations.
+
+Copyright: Copyright 2011-2019 The Error Prone Authors
+Home page: https://github.com/google/error-prone
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains checkerframework checker-qual Annotations.
+
+Copyright: 2004-2020 the Checker Framework developers
+Home page: https://github.com/typetools/checker-framework
+License: https://github.com/typetools/checker-framework/blob/master/LICENSE.txt (MIT license)
+
+License text:
+| The annotations are licensed under the MIT License. (The text of this
+| license appears below.) More specifically, all the parts of the Checker
+| Framework that you might want to include with your own program use the
+| MIT License. This is the checker-qual.jar file and all the files that
+| appear in it: every file in a qual/ directory, plus utility files such
+| as NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc.
+| In addition, the cleanroom implementations of third-party annotations,
+| which the Checker Framework recognizes as aliases for its own
+| annotations, are licensed under the MIT License.
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in
+| all copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+| THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Animal Sniffer Annotations.
+
+Copyright: 2009-2018 codehaus.org
+Home page: https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations/
+License: https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations/license.html (MIT license)
+
+License text:
+| The MIT License
+|
+| Copyright (c) 2009 codehaus.org.
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in
+| all copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+| THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Caffeine by Ben Manes.
+
+Copyright: 2014-2020 Ben Manes and contributors
+Home page: https://github.com/ben-manes/caffeine
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache Yetus audience annotations.
+
+Copyright: 2008-2020 The Apache Software Foundation.
+Home page: https://yetus.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Google protobuf.
+
+Copyright: 2008 Google Inc.
+Home page: https://developers.google.com/protocol-buffers
+License: https://github.com/protocolbuffers/protobuf/blob/master/LICENSE (BSD)
+
+License text:
+
+| Copyright 2008 Google Inc. All rights reserved.
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions are
+| met:
+|
+| * Redistributions of source code must retain the above copyright
+| notice, this list of conditions and the following disclaimer.
+| * Redistributions in binary form must reproduce the above
+| copyright notice, this list of conditions and the following disclaimer
+| in the documentation and/or other materials provided with the
+| distribution.
+| * Neither the name of Google Inc. nor the names of its
+| contributors may be used to endorse or promote products derived from
+| this software without specific prior written permission.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+|
+| Code generated by the Protocol Buffer compiler is owned by the owner
+| of the input file used when generating it. This code is not
+| standalone and requires a support library to be linked with it. This
+| support library is itself covered by the above license.
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains ThreeTen.
+
+Copyright: 2007-present, Stephen Colebourne & Michael Nascimento Santos.
+Home page: https://www.threeten.org/threeten-extra/
+License: https://github.com/ThreeTen/threeten-extra/blob/master/LICENSE.txt (BSD 3-clause)
+
+License text:
+
+| All rights reserved.
+|
+| * Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions are met:
+|
+| * Redistributions of source code must retain the above copyright notice,
+| this list of conditions and the following disclaimer.
+|
+| * Redistributions in binary form must reproduce the above copyright notice,
+| this list of conditions and the following disclaimer in the documentation
+| and/or other materials provided with the distribution.
+|
+| * Neither the name of JSR-310 nor the names of its contributors
+| may be used to endorse or promote products derived from this software
+| without specific prior written permission.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+| CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This binary artifact includes Project Nessie with the following in its NOTICE
+file:
+
+| Dremio
+| Copyright 2015-2017 Dremio Corporation
+|
+| This product includes software developed at
+| The Apache Software Foundation (http://www.apache.org/).
+
+--------------------------------------------------------------------------------
+
+This binary includes code from Apache Commons.
+
+* Core ArrayUtil.
+
+Copyright: 2020 The Apache Software Foundation
+Home page: https://commons.apache.org/
+License: https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache HttpComponents Client.
+
+Copyright: 1999-2022 The Apache Software Foundation.
+Home page: https://hc.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache HttpComponents Client.
+
+* retry and error handling logic in ExponentialHttpRequestRetryStrategy.java
+
+Copyright: 1999-2022 The Apache Software Foundation.
+Home page: https://hc.apache.org/
+License: https://www.apache.org/licenses/LICENSE-2.0
diff --git a/flink/v1.18/flink-runtime/NOTICE b/flink/v1.18/flink-runtime/NOTICE
new file mode 100644
index 0000000000..81aa166045
--- /dev/null
+++ b/flink/v1.18/flink-runtime/NOTICE
@@ -0,0 +1,91 @@
+
+Apache Iceberg
+Copyright 2017-2022 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+--------------------------------------------------------------------------------
+
+This binary artifact includes Apache ORC with the following in its NOTICE file:
+
+| Apache ORC
+| Copyright 2013-2019 The Apache Software Foundation
+|
+| This product includes software developed by The Apache Software
+| Foundation (http://www.apache.org/).
+|
+| This product includes software developed by Hewlett-Packard:
+| (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
+
+--------------------------------------------------------------------------------
+
+This binary artifact includes Airlift Aircompressor with the following in its
+NOTICE file:
+
+| Snappy Copyright Notices
+| =========================
+|
+| * Copyright 2011 Dain Sundstrom <da...@iq80.com>
+| * Copyright 2011, Google Inc.<op...@google.com>
+|
+|
+| Snappy License
+| ===============
+| Copyright 2011, Google Inc.
+| All rights reserved.
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions are
+| met:
+|
+| * Redistributions of source code must retain the above copyright
+| notice, this list of conditions and the following disclaimer.
+| * Redistributions in binary form must reproduce the above
+| copyright notice, this list of conditions and the following disclaimer
+| in the documentation and/or other materials provided with the
+| distribution.
+| * Neither the name of Google Inc. nor the names of its
+| contributors may be used to endorse or promote products derived from
+| this software without specific prior written permission.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This binary artifact includes Apache Yetus with the following in its NOTICE
+file:
+
+| Apache Yetus
+| Copyright 2008-2020 The Apache Software Foundation
+|
+| This product includes software developed at
+| The Apache Software Foundation (https://www.apache.org/).
+|
+| ---
+| Additional licenses for the Apache Yetus Source/Website:
+| ---
+|
+|
+| See LICENSE for terms.
+
+--------------------------------------------------------------------------------
+
+This binary artifact includes Project Nessie with the following in its NOTICE
+file:
+
+| Dremio
+| Copyright 2015-2017 Dremio Corporation
+|
+| This product includes software developed at
+| The Apache Software Foundation (http://www.apache.org/).
diff --git a/flink/v1.18/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java b/flink/v1.18/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java
new file mode 100644
index 0000000000..65cfba1ec8
--- /dev/null
+++ b/flink/v1.18/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.Map;
+
+public class IcebergConnectorSmokeTest extends TestIcebergConnector {
+
+ public IcebergConnectorSmokeTest(
+ String catalogName, Map<String, String> properties, boolean isStreaming) {
+ super(catalogName, properties, isStreaming);
+ }
+}
diff --git a/flink/v1.18/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java b/flink/v1.18/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java
new file mode 100644
index 0000000000..c391716575
--- /dev/null
+++ b/flink/v1.18/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.sink.shuffle;
+
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortKey;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.SingleShotTime)
+public class MapRangePartitionerBenchmark {
+ private static final String CHARS =
+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.!?";
+ private static final int SAMPLE_SIZE = 100_000;
+ private static final Schema SCHEMA =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.required(2, "name2", Types.StringType.get()),
+ Types.NestedField.required(3, "name3", Types.StringType.get()),
+ Types.NestedField.required(4, "name4", Types.StringType.get()),
+ Types.NestedField.required(5, "name5", Types.StringType.get()),
+ Types.NestedField.required(6, "name6", Types.StringType.get()),
+ Types.NestedField.required(7, "name7", Types.StringType.get()),
+ Types.NestedField.required(8, "name8", Types.StringType.get()),
+ Types.NestedField.required(9, "name9", Types.StringType.get()));
+
+ private static final SortOrder SORT_ORDER = SortOrder.builderFor(SCHEMA).asc("id").build();
+ private static final SortKey SORT_KEY = new SortKey(SCHEMA, SORT_ORDER);
+
+ private MapRangePartitioner partitioner;
+ private RowData[] rows;
+
+ @Setup
+ public void setupBenchmark() {
+ NavigableMap<Integer, Long> weights = longTailDistribution(100_000, 24, 240, 100, 2.0);
+ Map<SortKey, Long> mapStatistics = Maps.newHashMapWithExpectedSize(weights.size());
+ weights.forEach(
+ (id, weight) -> {
+ SortKey sortKey = SORT_KEY.copy();
+ sortKey.set(0, id);
+ mapStatistics.put(sortKey, weight);
+ });
+
+ MapDataStatistics dataStatistics = new MapDataStatistics(mapStatistics);
+ this.partitioner =
+ new MapRangePartitioner(
+ SCHEMA, SortOrder.builderFor(SCHEMA).asc("id").build(), dataStatistics, 2);
+
+ List<Integer> keys = Lists.newArrayList(weights.keySet().iterator());
+ long[] weightsCDF = new long[keys.size()];
+ long totalWeight = 0;
+ for (int i = 0; i < keys.size(); ++i) {
+ totalWeight += weights.get(keys.get(i));
+ weightsCDF[i] = totalWeight;
+ }
+
+ // pre-calculate the samples for benchmark run
+ this.rows = new GenericRowData[SAMPLE_SIZE];
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ long weight = ThreadLocalRandom.current().nextLong(totalWeight);
+ int index = binarySearchIndex(weightsCDF, weight);
+ rows[i] =
+ GenericRowData.of(
+ keys.get(index),
+ randomString("name2-"),
+ randomString("name3-"),
+ randomString("name4-"),
+ randomString("name5-"),
+ randomString("name6-"),
+ randomString("name7-"),
+ randomString("name8-"),
+ randomString("name9-"));
+ }
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {}
+
+ @Benchmark
+ @Threads(1)
+ public void testPartitionerLongTailDistribution(Blackhole blackhole) {
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ blackhole.consume(partitioner.partition(rows[i], 128));
+ }
+ }
+
+ private static String randomString(String prefix) {
+ int length = ThreadLocalRandom.current().nextInt(200);
+ byte[] buffer = new byte[length];
+
+ for (int i = 0; i < length; i += 1) {
+ buffer[i] = (byte) CHARS.charAt(ThreadLocalRandom.current().nextInt(CHARS.length()));
+ }
+
+ return prefix + new String(buffer);
+ }
+
+ /** find the index where weightsUDF[index] < weight && weightsUDF[index+1] >= weight */
+ private static int binarySearchIndex(long[] weightsUDF, long target) {
+ Preconditions.checkArgument(
+ target < weightsUDF[weightsUDF.length - 1],
+ "weight is out of range: total weight = %s, search target = %s",
+ weightsUDF[weightsUDF.length - 1],
+ target);
+ int start = 0;
+ int end = weightsUDF.length - 1;
+ while (start < end) {
+ int mid = (start + end) / 2;
+ if (weightsUDF[mid] < target && weightsUDF[mid + 1] >= target) {
+ return mid;
+ }
+
+ if (weightsUDF[mid] >= target) {
+ end = mid - 1;
+ } else if (weightsUDF[mid + 1] < target) {
+ start = mid + 1;
+ }
+ }
+ return start;
+ }
+
+ /** Key is the id string and value is the weight in long value. */
+ private static NavigableMap<Integer, Long> longTailDistribution(
+ long startingWeight,
+ int longTailStartingIndex,
+ int longTailLength,
+ long longTailBaseWeight,
+ double weightRandomJitterPercentage) {
+
+ NavigableMap<Integer, Long> weights = Maps.newTreeMap();
+
+ // first part just decays the weight by half
+ long currentWeight = startingWeight;
+ for (int index = 0; index < longTailStartingIndex; ++index) {
+ double jitter = ThreadLocalRandom.current().nextDouble(weightRandomJitterPercentage / 100);
+ long weight = (long) (currentWeight * (1.0 + jitter));
+ weight = weight > 0 ? weight : 1;
+ weights.put(index, weight);
+ if (currentWeight > longTailBaseWeight) {
+ currentWeight = currentWeight / 2;
+ }
+ }
+
+ // long tail part
+ for (int index = longTailStartingIndex;
+ index < longTailStartingIndex + longTailLength;
+ ++index) {
+ long longTailWeight =
+ (long)
+ (longTailBaseWeight
+ * ThreadLocalRandom.current().nextDouble(weightRandomJitterPercentage));
+ longTailWeight = longTailWeight > 0 ? longTailWeight : 1;
+ weights.put(index, longTailWeight);
+ }
+
+ return weights;
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java
new file mode 100644
index 0000000000..18473bf4f1
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.iceberg.hadoop.SerializableConfiguration;
+import org.apache.iceberg.hive.HiveCatalog;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.rest.RESTCatalog;
+
+/** Serializable loader to load an Iceberg {@link Catalog}. */
+public interface CatalogLoader extends Serializable, Cloneable {
+
+ /**
+ * Create a new catalog with the provided properties. NOTICE: for flink, we may initialize the
+ * {@link CatalogLoader} at flink sql client side or job manager side, and then serialize this
+ * catalog loader to task manager, finally deserialize it and create a new catalog at task manager
+ * side.
+ *
+ * @return a newly created {@link Catalog}
+ */
+ Catalog loadCatalog();
+
+ /** Clone a CatalogLoader. */
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ CatalogLoader clone();
+
+ static CatalogLoader hadoop(
+ String name, Configuration hadoopConf, Map<String, String> properties) {
+ return new HadoopCatalogLoader(name, hadoopConf, properties);
+ }
+
+ static CatalogLoader hive(String name, Configuration hadoopConf, Map<String, String> properties) {
+ return new HiveCatalogLoader(name, hadoopConf, properties);
+ }
+
+ static CatalogLoader rest(String name, Configuration hadoopConf, Map<String, String> properties) {
+ return new RESTCatalogLoader(name, hadoopConf, properties);
+ }
+
+ static CatalogLoader custom(
+ String name, Map<String, String> properties, Configuration hadoopConf, String impl) {
+ return new CustomCatalogLoader(name, properties, hadoopConf, impl);
+ }
+
+ class HadoopCatalogLoader implements CatalogLoader {
+ private final String catalogName;
+ private final SerializableConfiguration hadoopConf;
+ private final String warehouseLocation;
+ private final Map<String, String> properties;
+
+ private HadoopCatalogLoader(
+ String catalogName, Configuration conf, Map<String, String> properties) {
+ this.catalogName = catalogName;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.warehouseLocation = properties.get(CatalogProperties.WAREHOUSE_LOCATION);
+ this.properties = Maps.newHashMap(properties);
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(
+ HadoopCatalog.class.getName(), catalogName, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new HadoopCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("catalogName", catalogName)
+ .add("warehouseLocation", warehouseLocation)
+ .toString();
+ }
+ }
+
+ class HiveCatalogLoader implements CatalogLoader {
+ private final String catalogName;
+ private final SerializableConfiguration hadoopConf;
+ private final String uri;
+ private final String warehouse;
+ private final int clientPoolSize;
+ private final Map<String, String> properties;
+
+ private HiveCatalogLoader(
+ String catalogName, Configuration conf, Map<String, String> properties) {
+ this.catalogName = catalogName;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.uri = properties.get(CatalogProperties.URI);
+ this.warehouse = properties.get(CatalogProperties.WAREHOUSE_LOCATION);
+ this.clientPoolSize =
+ properties.containsKey(CatalogProperties.CLIENT_POOL_SIZE)
+ ? Integer.parseInt(properties.get(CatalogProperties.CLIENT_POOL_SIZE))
+ : CatalogProperties.CLIENT_POOL_SIZE_DEFAULT;
+ this.properties = Maps.newHashMap(properties);
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(
+ HiveCatalog.class.getName(), catalogName, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new HiveCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("catalogName", catalogName)
+ .add("uri", uri)
+ .add("warehouse", warehouse)
+ .add("clientPoolSize", clientPoolSize)
+ .toString();
+ }
+ }
+
+ class RESTCatalogLoader implements CatalogLoader {
+ private final String catalogName;
+ private final SerializableConfiguration hadoopConf;
+ private final Map<String, String> properties;
+
+ private RESTCatalogLoader(
+ String catalogName, Configuration conf, Map<String, String> properties) {
+ this.catalogName = catalogName;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.properties = Maps.newHashMap(properties);
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(
+ RESTCatalog.class.getName(), catalogName, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new RESTCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("catalogName", catalogName)
+ .add("properties", properties)
+ .toString();
+ }
+ }
+
+ class CustomCatalogLoader implements CatalogLoader {
+
+ private final SerializableConfiguration hadoopConf;
+ private final Map<String, String> properties;
+ private final String name;
+ private final String impl;
+
+ private CustomCatalogLoader(
+ String name, Map<String, String> properties, Configuration conf, String impl) {
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.properties = Maps.newHashMap(properties); // wrap into a hashmap for serialization
+ this.name = name;
+ this.impl =
+ Preconditions.checkNotNull(
+ impl, "Cannot initialize custom Catalog, impl class name is null");
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(impl, name, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new CustomCatalogLoader(name, properties, new Configuration(hadoopConf.get()), impl);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("name", name).add("impl", impl).toString();
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
new file mode 100644
index 0000000000..86295d78cc
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
@@ -0,0 +1,833 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.catalog.AbstractCatalog;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogDatabase;
+import org.apache.flink.table.catalog.CatalogDatabaseImpl;
+import org.apache.flink.table.catalog.CatalogFunction;
+import org.apache.flink.table.catalog.CatalogPartition;
+import org.apache.flink.table.catalog.CatalogPartitionSpec;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.CatalogTableImpl;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.TableChange;
+import org.apache.flink.table.catalog.exceptions.CatalogException;
+import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
+import org.apache.flink.table.catalog.exceptions.FunctionNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException;
+import org.apache.flink.table.catalog.stats.CatalogColumnStatistics;
+import org.apache.flink.table.catalog.stats.CatalogTableStatistics;
+import org.apache.flink.table.expressions.Expression;
+import org.apache.flink.table.factories.Factory;
+import org.apache.flink.util.StringUtils;
+import org.apache.iceberg.CachingCatalog;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.MetadataTableType;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.flink.util.FlinkAlterTableUtil;
+import org.apache.iceberg.flink.util.FlinkCompatibilityUtil;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Splitter;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+
+/**
+ * A Flink Catalog implementation that wraps an Iceberg {@link Catalog}.
+ *
+ * <p>The mapping between Flink database and Iceberg namespace: Supplying a base namespace for a
+ * given catalog, so if you have a catalog that supports a 2-level namespace, you would supply the
+ * first level in the catalog configuration and the second level would be exposed as Flink
+ * databases.
+ *
+ * <p>The Iceberg table manages its partitions by itself. The partition of the Iceberg table is
+ * independent of the partition of Flink.
+ */
+public class FlinkCatalog extends AbstractCatalog {
+ private final CatalogLoader catalogLoader;
+ private final Catalog icebergCatalog;
+ private final Namespace baseNamespace;
+ private final SupportsNamespaces asNamespaceCatalog;
+ private final Closeable closeable;
+ private final boolean cacheEnabled;
+
+ public FlinkCatalog(
+ String catalogName,
+ String defaultDatabase,
+ Namespace baseNamespace,
+ CatalogLoader catalogLoader,
+ boolean cacheEnabled,
+ long cacheExpirationIntervalMs) {
+ super(catalogName, defaultDatabase);
+ this.catalogLoader = catalogLoader;
+ this.baseNamespace = baseNamespace;
+ this.cacheEnabled = cacheEnabled;
+
+ Catalog originalCatalog = catalogLoader.loadCatalog();
+ icebergCatalog =
+ cacheEnabled
+ ? CachingCatalog.wrap(originalCatalog, cacheExpirationIntervalMs)
+ : originalCatalog;
+ asNamespaceCatalog =
+ originalCatalog instanceof SupportsNamespaces ? (SupportsNamespaces) originalCatalog : null;
+ closeable = originalCatalog instanceof Closeable ? (Closeable) originalCatalog : null;
+
+ FlinkEnvironmentContext.init();
+ }
+
+ @Override
+ public void open() throws CatalogException {}
+
+ @Override
+ public void close() throws CatalogException {
+ if (closeable != null) {
+ try {
+ closeable.close();
+ } catch (IOException e) {
+ throw new CatalogException(e);
+ }
+ }
+ }
+
+ public Catalog catalog() {
+ return icebergCatalog;
+ }
+
+ /** Append a new level to the base namespace */
+ private static Namespace appendLevel(Namespace baseNamespace, String newLevel) {
+ String[] namespace = new String[baseNamespace.levels().length + 1];
+ System.arraycopy(baseNamespace.levels(), 0, namespace, 0, baseNamespace.levels().length);
+ namespace[baseNamespace.levels().length] = newLevel;
+ return Namespace.of(namespace);
+ }
+
+ TableIdentifier toIdentifier(ObjectPath path) {
+ String objectName = path.getObjectName();
+ List<String> tableName = Splitter.on('$').splitToList(objectName);
+
+ if (tableName.size() == 1) {
+ return TableIdentifier.of(
+ appendLevel(baseNamespace, path.getDatabaseName()), path.getObjectName());
+ } else if (tableName.size() == 2 && MetadataTableType.from(tableName.get(1)) != null) {
+ return TableIdentifier.of(
+ appendLevel(appendLevel(baseNamespace, path.getDatabaseName()), tableName.get(0)),
+ tableName.get(1));
+ } else {
+ throw new IllegalArgumentException("Illegal table name:" + objectName);
+ }
+ }
+
+ @Override
+ public List<String> listDatabases() throws CatalogException {
+ if (asNamespaceCatalog == null) {
+ return Collections.singletonList(getDefaultDatabase());
+ }
+
+ return asNamespaceCatalog.listNamespaces(baseNamespace).stream()
+ .map(n -> n.level(n.levels().length - 1))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public CatalogDatabase getDatabase(String databaseName)
+ throws DatabaseNotExistException, CatalogException {
+ if (asNamespaceCatalog == null) {
+ if (!getDefaultDatabase().equals(databaseName)) {
+ throw new DatabaseNotExistException(getName(), databaseName);
+ } else {
+ return new CatalogDatabaseImpl(Maps.newHashMap(), "");
+ }
+ } else {
+ try {
+ Map<String, String> metadata =
+ Maps.newHashMap(
+ asNamespaceCatalog.loadNamespaceMetadata(appendLevel(baseNamespace, databaseName)));
+ String comment = metadata.remove("comment");
+ return new CatalogDatabaseImpl(metadata, comment);
+ } catch (NoSuchNamespaceException e) {
+ throw new DatabaseNotExistException(getName(), databaseName, e);
+ }
+ }
+ }
+
+ @Override
+ public boolean databaseExists(String databaseName) throws CatalogException {
+ try {
+ getDatabase(databaseName);
+ return true;
+ } catch (DatabaseNotExistException ignore) {
+ return false;
+ }
+ }
+
+ @Override
+ public void createDatabase(String name, CatalogDatabase database, boolean ignoreIfExists)
+ throws DatabaseAlreadyExistException, CatalogException {
+ createDatabase(
+ name, mergeComment(database.getProperties(), database.getComment()), ignoreIfExists);
+ }
+
+ private void createDatabase(
+ String databaseName, Map<String, String> metadata, boolean ignoreIfExists)
+ throws DatabaseAlreadyExistException, CatalogException {
+ if (asNamespaceCatalog != null) {
+ try {
+ asNamespaceCatalog.createNamespace(appendLevel(baseNamespace, databaseName), metadata);
+ } catch (AlreadyExistsException e) {
+ if (!ignoreIfExists) {
+ throw new DatabaseAlreadyExistException(getName(), databaseName, e);
+ }
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ "Namespaces are not supported by catalog: " + getName());
+ }
+ }
+
+ private Map<String, String> mergeComment(Map<String, String> metadata, String comment) {
+ Map<String, String> ret = Maps.newHashMap(metadata);
+ if (metadata.containsKey("comment")) {
+ throw new CatalogException("Database properties should not contain key: 'comment'.");
+ }
+
+ if (!StringUtils.isNullOrWhitespaceOnly(comment)) {
+ ret.put("comment", comment);
+ }
+ return ret;
+ }
+
+ @Override
+ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade)
+ throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException {
+ if (asNamespaceCatalog != null) {
+ try {
+ boolean success = asNamespaceCatalog.dropNamespace(appendLevel(baseNamespace, name));
+ if (!success && !ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name);
+ }
+ } catch (NoSuchNamespaceException e) {
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name, e);
+ }
+ } catch (NamespaceNotEmptyException e) {
+ throw new DatabaseNotEmptyException(getName(), name, e);
+ }
+ } else {
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name);
+ }
+ }
+ }
+
+ @Override
+ public void alterDatabase(String name, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
+ throws DatabaseNotExistException, CatalogException {
+ if (asNamespaceCatalog != null) {
+ Namespace namespace = appendLevel(baseNamespace, name);
+ Map<String, String> updates = Maps.newHashMap();
+ Set<String> removals = Sets.newHashSet();
+
+ try {
+ Map<String, String> oldProperties = asNamespaceCatalog.loadNamespaceMetadata(namespace);
+ Map<String, String> newProperties =
+ mergeComment(newDatabase.getProperties(), newDatabase.getComment());
+
+ for (String key : oldProperties.keySet()) {
+ if (!newProperties.containsKey(key)) {
+ removals.add(key);
+ }
+ }
+
+ for (Map.Entry<String, String> entry : newProperties.entrySet()) {
+ if (!entry.getValue().equals(oldProperties.get(entry.getKey()))) {
+ updates.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ if (!updates.isEmpty()) {
+ asNamespaceCatalog.setProperties(namespace, updates);
+ }
+
+ if (!removals.isEmpty()) {
+ asNamespaceCatalog.removeProperties(namespace, removals);
+ }
+
+ } catch (NoSuchNamespaceException e) {
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name, e);
+ }
+ }
+ } else {
+ if (getDefaultDatabase().equals(name)) {
+ throw new CatalogException(
+ "Can not alter the default database when the iceberg catalog doesn't support namespaces.");
+ }
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name);
+ }
+ }
+ }
+
+ @Override
+ public List<String> listTables(String databaseName)
+ throws DatabaseNotExistException, CatalogException {
+ try {
+ return icebergCatalog.listTables(appendLevel(baseNamespace, databaseName)).stream()
+ .map(TableIdentifier::name)
+ .collect(Collectors.toList());
+ } catch (NoSuchNamespaceException e) {
+ throw new DatabaseNotExistException(getName(), databaseName, e);
+ }
+ }
+
+ @Override
+ public CatalogTable getTable(ObjectPath tablePath)
+ throws TableNotExistException, CatalogException {
+ Table table = loadIcebergTable(tablePath);
+ return toCatalogTable(table);
+ }
+
+ private Table loadIcebergTable(ObjectPath tablePath) throws TableNotExistException {
+ try {
+ Table table = icebergCatalog.loadTable(toIdentifier(tablePath));
+ if (cacheEnabled) {
+ table.refresh();
+ }
+
+ return table;
+ } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
+ throw new TableNotExistException(getName(), tablePath, e);
+ }
+ }
+
+ @Override
+ public boolean tableExists(ObjectPath tablePath) throws CatalogException {
+ return icebergCatalog.tableExists(toIdentifier(tablePath));
+ }
+
+ @Override
+ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ try {
+ icebergCatalog.dropTable(toIdentifier(tablePath));
+ } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
+ if (!ignoreIfNotExists) {
+ throw new TableNotExistException(getName(), tablePath, e);
+ }
+ }
+ }
+
+ @Override
+ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
+ throws TableNotExistException, TableAlreadyExistException, CatalogException {
+ try {
+ icebergCatalog.renameTable(
+ toIdentifier(tablePath),
+ toIdentifier(new ObjectPath(tablePath.getDatabaseName(), newTableName)));
+ } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
+ if (!ignoreIfNotExists) {
+ throw new TableNotExistException(getName(), tablePath, e);
+ }
+ } catch (AlreadyExistsException e) {
+ throw new TableAlreadyExistException(getName(), tablePath, e);
+ }
+ }
+
+ @Override
+ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
+ throws CatalogException, TableAlreadyExistException {
+ if (Objects.equals(
+ table.getOptions().get("connector"), FlinkDynamicTableFactory.FACTORY_IDENTIFIER)) {
+ throw new IllegalArgumentException(
+ "Cannot create the table with 'connector'='iceberg' table property in "
+ + "an iceberg catalog, Please create table with 'connector'='iceberg' property in a non-iceberg catalog or "
+ + "create table without 'connector'='iceberg' related properties in an iceberg table.");
+ }
+ Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved");
+ createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists);
+ }
+
+ void createIcebergTable(ObjectPath tablePath, ResolvedCatalogTable table, boolean ignoreIfExists)
+ throws CatalogException, TableAlreadyExistException {
+ validateFlinkTable(table);
+
+ Schema icebergSchema = FlinkSchemaUtil.convert(table.getResolvedSchema());
+ PartitionSpec spec = toPartitionSpec(((CatalogTable) table).getPartitionKeys(), icebergSchema);
+ ImmutableMap.Builder<String, String> properties = ImmutableMap.builder();
+ String location = null;
+ for (Map.Entry<String, String> entry : table.getOptions().entrySet()) {
+ if ("location".equalsIgnoreCase(entry.getKey())) {
+ location = entry.getValue();
+ } else {
+ properties.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ try {
+ icebergCatalog.createTable(
+ toIdentifier(tablePath), icebergSchema, spec, location, properties.build());
+ } catch (AlreadyExistsException e) {
+ if (!ignoreIfExists) {
+ throw new TableAlreadyExistException(getName(), tablePath, e);
+ }
+ }
+ }
+
+ private static void validateTableSchemaAndPartition(CatalogTable ct1, CatalogTable ct2) {
+ TableSchema ts1 = ct1.getSchema();
+ TableSchema ts2 = ct2.getSchema();
+ boolean equalsPrimary = false;
+
+ if (ts1.getPrimaryKey().isPresent() && ts2.getPrimaryKey().isPresent()) {
+ equalsPrimary =
+ Objects.equals(ts1.getPrimaryKey().get().getType(), ts2.getPrimaryKey().get().getType())
+ && Objects.equals(
+ ts1.getPrimaryKey().get().getColumns(), ts2.getPrimaryKey().get().getColumns());
+ } else if (!ts1.getPrimaryKey().isPresent() && !ts2.getPrimaryKey().isPresent()) {
+ equalsPrimary = true;
+ }
+
+ if (!(Objects.equals(ts1.getTableColumns(), ts2.getTableColumns())
+ && Objects.equals(ts1.getWatermarkSpecs(), ts2.getWatermarkSpecs())
+ && equalsPrimary)) {
+ throw new UnsupportedOperationException(
+ "Altering schema is not supported in the old alterTable API. "
+ + "To alter schema, use the other alterTable API and provide a list of TableChange's.");
+ }
+
+ validateTablePartition(ct1, ct2);
+ }
+
+ private static void validateTablePartition(CatalogTable ct1, CatalogTable ct2) {
+ if (!ct1.getPartitionKeys().equals(ct2.getPartitionKeys())) {
+ throw new UnsupportedOperationException("Altering partition keys is not supported yet.");
+ }
+ }
+
+ /**
+ * This alterTable API only supports altering table properties.
+ *
+ * <p>Support for adding/removing/renaming columns cannot be done by comparing CatalogTable
+ * instances, unless the Flink schema contains Iceberg column IDs.
+ *
+ * <p>To alter columns, use the other alterTable API and provide a list of TableChange's.
+ *
+ * @param tablePath path of the table or view to be modified
+ * @param newTable the new table definition
+ * @param ignoreIfNotExists flag to specify behavior when the table or view does not exist: if set
+ * to false, throw an exception, if set to true, do nothing.
+ * @throws CatalogException in case of any runtime exception
+ * @throws TableNotExistException if the table does not exist
+ */
+ @Override
+ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
+ throws CatalogException, TableNotExistException {
+ validateFlinkTable(newTable);
+
+ Table icebergTable;
+ try {
+ icebergTable = loadIcebergTable(tablePath);
+ } catch (TableNotExistException e) {
+ if (!ignoreIfNotExists) {
+ throw e;
+ } else {
+ return;
+ }
+ }
+
+ CatalogTable table = toCatalogTable(icebergTable);
+ validateTableSchemaAndPartition(table, (CatalogTable) newTable);
+
+ Map<String, String> oldProperties = table.getOptions();
+ Map<String, String> setProperties = Maps.newHashMap();
+
+ String setLocation = null;
+ String setSnapshotId = null;
+ String pickSnapshotId = null;
+
+ for (Map.Entry<String, String> entry : newTable.getOptions().entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (Objects.equals(value, oldProperties.get(key))) {
+ continue;
+ }
+
+ if ("location".equalsIgnoreCase(key)) {
+ setLocation = value;
+ } else if ("current-snapshot-id".equalsIgnoreCase(key)) {
+ setSnapshotId = value;
+ } else if ("cherry-pick-snapshot-id".equalsIgnoreCase(key)) {
+ pickSnapshotId = value;
+ } else {
+ setProperties.put(key, value);
+ }
+ }
+
+ oldProperties
+ .keySet()
+ .forEach(
+ k -> {
+ if (!newTable.getOptions().containsKey(k)) {
+ setProperties.put(k, null);
+ }
+ });
+
+ FlinkAlterTableUtil.commitChanges(
+ icebergTable, setLocation, setSnapshotId, pickSnapshotId, setProperties);
+ }
+
+ @Override
+ public void alterTable(
+ ObjectPath tablePath,
+ CatalogBaseTable newTable,
+ List<TableChange> tableChanges,
+ boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ validateFlinkTable(newTable);
+
+ Table icebergTable;
+ try {
+ icebergTable = loadIcebergTable(tablePath);
+ } catch (TableNotExistException e) {
+ if (!ignoreIfNotExists) {
+ throw e;
+ } else {
+ return;
+ }
+ }
+
+ // Does not support altering partition yet.
+ validateTablePartition(toCatalogTable(icebergTable), (CatalogTable) newTable);
+
+ String setLocation = null;
+ String setSnapshotId = null;
+ String cherrypickSnapshotId = null;
+
+ List<TableChange> propertyChanges = Lists.newArrayList();
+ List<TableChange> schemaChanges = Lists.newArrayList();
+ for (TableChange change : tableChanges) {
+ if (change instanceof TableChange.SetOption) {
+ TableChange.SetOption set = (TableChange.SetOption) change;
+
+ if ("location".equalsIgnoreCase(set.getKey())) {
+ setLocation = set.getValue();
+ } else if ("current-snapshot-id".equalsIgnoreCase(set.getKey())) {
+ setSnapshotId = set.getValue();
+ } else if ("cherry-pick-snapshot-id".equalsIgnoreCase(set.getKey())) {
+ cherrypickSnapshotId = set.getValue();
+ } else {
+ propertyChanges.add(change);
+ }
+ } else if (change instanceof TableChange.ResetOption) {
+ propertyChanges.add(change);
+ } else {
+ schemaChanges.add(change);
+ }
+ }
+
+ FlinkAlterTableUtil.commitChanges(
+ icebergTable,
+ setLocation,
+ setSnapshotId,
+ cherrypickSnapshotId,
+ schemaChanges,
+ propertyChanges);
+ }
+
+ private static void validateFlinkTable(CatalogBaseTable table) {
+ Preconditions.checkArgument(
+ table instanceof CatalogTable, "The Table should be a CatalogTable.");
+
+ TableSchema schema = table.getSchema();
+ schema
+ .getTableColumns()
+ .forEach(
+ column -> {
+ if (!FlinkCompatibilityUtil.isPhysicalColumn(column)) {
+ throw new UnsupportedOperationException(
+ "Creating table with computed columns is not supported yet.");
+ }
+ });
+
+ if (!schema.getWatermarkSpecs().isEmpty()) {
+ throw new UnsupportedOperationException(
+ "Creating table with watermark specs is not supported yet.");
+ }
+ }
+
+ private static PartitionSpec toPartitionSpec(List<String> partitionKeys, Schema icebergSchema) {
+ PartitionSpec.Builder builder = PartitionSpec.builderFor(icebergSchema);
+ partitionKeys.forEach(builder::identity);
+ return builder.build();
+ }
+
+ private static List<String> toPartitionKeys(PartitionSpec spec, Schema icebergSchema) {
+ ImmutableList.Builder<String> partitionKeysBuilder = ImmutableList.builder();
+ for (PartitionField field : spec.fields()) {
+ if (field.transform().isIdentity()) {
+ partitionKeysBuilder.add(icebergSchema.findColumnName(field.sourceId()));
+ } else {
+ // Not created by Flink SQL.
+ // For compatibility with iceberg tables, return empty.
+ // TODO modify this after Flink support partition transform.
+ return Collections.emptyList();
+ }
+ }
+ return partitionKeysBuilder.build();
+ }
+
+ static CatalogTable toCatalogTable(Table table) {
+ TableSchema schema = FlinkSchemaUtil.toSchema(table.schema());
+ List<String> partitionKeys = toPartitionKeys(table.spec(), table.schema());
+
+ // NOTE: We can not create a IcebergCatalogTable extends CatalogTable, because Flink optimizer
+ // may use
+ // CatalogTableImpl to copy a new catalog table.
+ // Let's re-loading table from Iceberg catalog when creating source/sink operators.
+ // Iceberg does not have Table comment, so pass a null (Default comment value in Flink).
+ return new CatalogTableImpl(schema, partitionKeys, table.properties(), null);
+ }
+
+ @Override
+ public Optional<Factory> getFactory() {
+ return Optional.of(new FlinkDynamicTableFactory(this));
+ }
+
+ CatalogLoader getCatalogLoader() {
+ return catalogLoader;
+ }
+
+ // ------------------------------ Unsupported methods
+ // ---------------------------------------------
+
+ @Override
+ public List<String> listViews(String databaseName) throws CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void createPartition(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogPartition partition,
+ boolean ignoreIfExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropPartition(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartition(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogPartition newPartition,
+ boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<String> listFunctions(String dbName) throws CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CatalogFunction getFunction(ObjectPath functionPath)
+ throws FunctionNotExistException, CatalogException {
+ throw new FunctionNotExistException(getName(), functionPath);
+ }
+
+ @Override
+ public boolean functionExists(ObjectPath functionPath) throws CatalogException {
+ return false;
+ }
+
+ @Override
+ public void createFunction(
+ ObjectPath functionPath, CatalogFunction function, boolean ignoreIfExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterFunction(
+ ObjectPath functionPath, CatalogFunction newFunction, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterTableStatistics(
+ ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterTableColumnStatistics(
+ ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartitionStatistics(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogTableStatistics partitionStatistics,
+ boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartitionColumnStatistics(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogColumnStatistics columnStatistics,
+ boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
+ throws TableNotExistException, TableNotPartitionedException, CatalogException {
+ Table table = loadIcebergTable(tablePath);
+
+ if (table.spec().isUnpartitioned()) {
+ throw new TableNotPartitionedException(icebergCatalog.name(), tablePath);
+ }
+
+ Set<CatalogPartitionSpec> set = Sets.newHashSet();
+ try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
+ for (DataFile dataFile : CloseableIterable.transform(tasks, FileScanTask::file)) {
+ Map<String, String> map = Maps.newHashMap();
+ StructLike structLike = dataFile.partition();
+ PartitionSpec spec = table.specs().get(dataFile.specId());
+ for (int i = 0; i < structLike.size(); i++) {
+ map.put(spec.fields().get(i).name(), String.valueOf(structLike.get(i, Object.class)));
+ }
+ set.add(new CatalogPartitionSpec(map));
+ }
+ } catch (IOException e) {
+ throw new CatalogException(
+ String.format("Failed to list partitions of table %s", tablePath), e);
+ }
+
+ return Lists.newArrayList(set);
+ }
+
+ @Override
+ public List<CatalogPartitionSpec> listPartitions(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<CatalogPartitionSpec> listPartitionsByFilter(
+ ObjectPath tablePath, List<Expression> filters) throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ // After partition pruning and filter push down, the statistics have become very inaccurate, so
+ // the statistics from
+ // here are of little significance.
+ // Flink will support something like SupportsReportStatistics in future.
+
+ @Override
+ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws CatalogException {
+ return CatalogTableStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath)
+ throws CatalogException {
+ return CatalogColumnStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogTableStatistics getPartitionStatistics(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
+ return CatalogTableStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogColumnStatistics getPartitionColumnStatistics(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
+ return CatalogColumnStatistics.UNKNOWN;
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java
new file mode 100644
index 0000000000..1453753849
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.runtime.util.HadoopUtils;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.factories.CatalogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.PropertyUtil;
+
+/**
+ * A Flink Catalog factory implementation that creates {@link FlinkCatalog}.
+ *
+ * <p>This supports the following catalog configuration options:
+ *
+ * <ul>
+ * <li><code>type</code> - Flink catalog factory key, should be "iceberg"
+ * <li><code>catalog-type</code> - iceberg catalog type, "hive", "hadoop" or "rest"
+ * <li><code>uri</code> - the Hive Metastore URI (Hive catalog only)
+ * <li><code>clients</code> - the Hive Client Pool Size (Hive catalog only)
+ * <li><code>warehouse</code> - the warehouse path (Hadoop catalog only)
+ * <li><code>default-database</code> - a database name to use as the default
+ * <li><code>base-namespace</code> - a base namespace as the prefix for all databases (Hadoop
+ * catalog only)
+ * <li><code>cache-enabled</code> - whether to enable catalog cache
+ * </ul>
+ *
+ * <p>To use a custom catalog that is not a Hive or Hadoop catalog, extend this class and override
+ * {@link #createCatalogLoader(String, Map, Configuration)}.
+ */
+public class FlinkCatalogFactory implements CatalogFactory {
+
+ // Can not just use "type", it conflicts with CATALOG_TYPE.
+ public static final String ICEBERG_CATALOG_TYPE = "catalog-type";
+ public static final String ICEBERG_CATALOG_TYPE_HADOOP = "hadoop";
+ public static final String ICEBERG_CATALOG_TYPE_HIVE = "hive";
+ public static final String ICEBERG_CATALOG_TYPE_REST = "rest";
+
+ public static final String HIVE_CONF_DIR = "hive-conf-dir";
+ public static final String HADOOP_CONF_DIR = "hadoop-conf-dir";
+ public static final String DEFAULT_DATABASE = "default-database";
+ public static final String DEFAULT_DATABASE_NAME = "default";
+ public static final String BASE_NAMESPACE = "base-namespace";
+
+ public static final String TYPE = "type";
+ public static final String PROPERTY_VERSION = "property-version";
+
+ /**
+ * Create an Iceberg {@link org.apache.iceberg.catalog.Catalog} loader to be used by this Flink
+ * catalog adapter.
+ *
+ * @param name Flink's catalog name
+ * @param properties Flink's catalog properties
+ * @param hadoopConf Hadoop configuration for catalog
+ * @return an Iceberg catalog loader
+ */
+ static CatalogLoader createCatalogLoader(
+ String name, Map<String, String> properties, Configuration hadoopConf) {
+ String catalogImpl = properties.get(CatalogProperties.CATALOG_IMPL);
+ if (catalogImpl != null) {
+ String catalogType = properties.get(ICEBERG_CATALOG_TYPE);
+ Preconditions.checkArgument(
+ catalogType == null,
+ "Cannot create catalog %s, both catalog-type and catalog-impl are set: catalog-type=%s, catalog-impl=%s",
+ name,
+ catalogType,
+ catalogImpl);
+ return CatalogLoader.custom(name, properties, hadoopConf, catalogImpl);
+ }
+
+ String catalogType = properties.getOrDefault(ICEBERG_CATALOG_TYPE, ICEBERG_CATALOG_TYPE_HIVE);
+ switch (catalogType.toLowerCase(Locale.ENGLISH)) {
+ case ICEBERG_CATALOG_TYPE_HIVE:
+ // The values of properties 'uri', 'warehouse', 'hive-conf-dir' are allowed to be null, in
+ // that case it will
+ // fallback to parse those values from hadoop configuration which is loaded from classpath.
+ String hiveConfDir = properties.get(HIVE_CONF_DIR);
+ String hadoopConfDir = properties.get(HADOOP_CONF_DIR);
+ Configuration newHadoopConf = mergeHiveConf(hadoopConf, hiveConfDir, hadoopConfDir);
+ return CatalogLoader.hive(name, newHadoopConf, properties);
+
+ case ICEBERG_CATALOG_TYPE_HADOOP:
+ return CatalogLoader.hadoop(name, hadoopConf, properties);
+
+ case ICEBERG_CATALOG_TYPE_REST:
+ return CatalogLoader.rest(name, hadoopConf, properties);
+
+ default:
+ throw new UnsupportedOperationException(
+ "Unknown catalog-type: " + catalogType + " (Must be 'hive', 'hadoop' or 'rest')");
+ }
+ }
+
+ @Override
+ public Map<String, String> requiredContext() {
+ Map<String, String> context = Maps.newHashMap();
+ context.put(TYPE, "iceberg");
+ context.put(PROPERTY_VERSION, "1");
+ return context;
+ }
+
+ @Override
+ public List<String> supportedProperties() {
+ return ImmutableList.of("*");
+ }
+
+ @Override
+ public Catalog createCatalog(String name, Map<String, String> properties) {
+ return createCatalog(name, properties, clusterHadoopConf());
+ }
+
+ protected Catalog createCatalog(
+ String name, Map<String, String> properties, Configuration hadoopConf) {
+ CatalogLoader catalogLoader = createCatalogLoader(name, properties, hadoopConf);
+ String defaultDatabase = properties.getOrDefault(DEFAULT_DATABASE, DEFAULT_DATABASE_NAME);
+
+ Namespace baseNamespace = Namespace.empty();
+ if (properties.containsKey(BASE_NAMESPACE)) {
+ baseNamespace = Namespace.of(properties.get(BASE_NAMESPACE).split("\\."));
+ }
+
+ boolean cacheEnabled =
+ PropertyUtil.propertyAsBoolean(
+ properties, CatalogProperties.CACHE_ENABLED, CatalogProperties.CACHE_ENABLED_DEFAULT);
+
+ long cacheExpirationIntervalMs =
+ PropertyUtil.propertyAsLong(
+ properties,
+ CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS,
+ CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS_OFF);
+ Preconditions.checkArgument(
+ cacheExpirationIntervalMs != 0,
+ "%s is not allowed to be 0.",
+ CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS);
+
+ return new FlinkCatalog(
+ name,
+ defaultDatabase,
+ baseNamespace,
+ catalogLoader,
+ cacheEnabled,
+ cacheExpirationIntervalMs);
+ }
+
+ private static Configuration mergeHiveConf(
+ Configuration hadoopConf, String hiveConfDir, String hadoopConfDir) {
+ Configuration newConf = new Configuration(hadoopConf);
+ if (!Strings.isNullOrEmpty(hiveConfDir)) {
+ Preconditions.checkState(
+ Files.exists(Paths.get(hiveConfDir, "hive-site.xml")),
+ "There should be a hive-site.xml file under the directory %s",
+ hiveConfDir);
+ newConf.addResource(new Path(hiveConfDir, "hive-site.xml"));
+ } else {
+ // If don't provide the hive-site.xml path explicitly, it will try to load resource from
+ // classpath. If still
+ // couldn't load the configuration file, then it will throw exception in HiveCatalog.
+ URL configFile = CatalogLoader.class.getClassLoader().getResource("hive-site.xml");
+ if (configFile != null) {
+ newConf.addResource(configFile);
+ }
+ }
+
+ if (!Strings.isNullOrEmpty(hadoopConfDir)) {
+ Preconditions.checkState(
+ Files.exists(Paths.get(hadoopConfDir, "hdfs-site.xml")),
+ "Failed to load Hadoop configuration: missing %s",
+ Paths.get(hadoopConfDir, "hdfs-site.xml"));
+ newConf.addResource(new Path(hadoopConfDir, "hdfs-site.xml"));
+ Preconditions.checkState(
+ Files.exists(Paths.get(hadoopConfDir, "core-site.xml")),
+ "Failed to load Hadoop configuration: missing %s",
+ Paths.get(hadoopConfDir, "core-site.xml"));
+ newConf.addResource(new Path(hadoopConfDir, "core-site.xml"));
+ }
+
+ return newConf;
+ }
+
+ public static Configuration clusterHadoopConf() {
+ return HadoopUtils.getHadoopConfiguration(GlobalConfiguration.loadConfiguration());
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java
new file mode 100644
index 0000000000..7167859e60
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.util.TimeUtils;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+
+class FlinkConfParser {
+
+ private final Map<String, String> tableProperties;
+ private final Map<String, String> options;
+ private final ReadableConfig readableConfig;
+
+ FlinkConfParser(Table table, Map<String, String> options, ReadableConfig readableConfig) {
+ this.tableProperties = table.properties();
+ this.options = options;
+ this.readableConfig = readableConfig;
+ }
+
+ public BooleanConfParser booleanConf() {
+ return new BooleanConfParser();
+ }
+
+ public IntConfParser intConf() {
+ return new IntConfParser();
+ }
+
+ public LongConfParser longConf() {
+ return new LongConfParser();
+ }
+
+ public <E extends Enum<E>> EnumConfParser<E> enumConfParser(Class<E> enumClass) {
+ return new EnumConfParser<>(enumClass);
+ }
+
+ public StringConfParser stringConf() {
+ return new StringConfParser();
+ }
+
+ public DurationConfParser durationConf() {
+ return new DurationConfParser();
+ }
+
+ class BooleanConfParser extends ConfParser<BooleanConfParser, Boolean> {
+ private Boolean defaultValue;
+
+ @Override
+ protected BooleanConfParser self() {
+ return this;
+ }
+
+ public BooleanConfParser defaultValue(boolean value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public BooleanConfParser defaultValue(String value) {
+ this.defaultValue = Boolean.parseBoolean(value);
+ return self();
+ }
+
+ public boolean parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Boolean::parseBoolean, defaultValue);
+ }
+ }
+
+ class IntConfParser extends ConfParser<IntConfParser, Integer> {
+ private Integer defaultValue;
+
+ @Override
+ protected IntConfParser self() {
+ return this;
+ }
+
+ public IntConfParser defaultValue(int value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public int parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Integer::parseInt, defaultValue);
+ }
+
+ public Integer parseOptional() {
+ return parse(Integer::parseInt, null);
+ }
+ }
+
+ class LongConfParser extends ConfParser<LongConfParser, Long> {
+ private Long defaultValue;
+
+ @Override
+ protected LongConfParser self() {
+ return this;
+ }
+
+ public LongConfParser defaultValue(long value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public long parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Long::parseLong, defaultValue);
+ }
+
+ public Long parseOptional() {
+ return parse(Long::parseLong, null);
+ }
+ }
+
+ class StringConfParser extends ConfParser<StringConfParser, String> {
+ private String defaultValue;
+
+ @Override
+ protected StringConfParser self() {
+ return this;
+ }
+
+ public StringConfParser defaultValue(String value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public String parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Function.identity(), defaultValue);
+ }
+
+ public String parseOptional() {
+ return parse(Function.identity(), null);
+ }
+ }
+
+ class EnumConfParser<E extends Enum<E>> extends ConfParser<EnumConfParser<E>, E> {
+ private E defaultValue;
+ private final Class<E> enumClass;
+
+ EnumConfParser(Class<E> enumClass) {
+ this.enumClass = enumClass;
+ }
+
+ @Override
+ protected EnumConfParser<E> self() {
+ return this;
+ }
+
+ public EnumConfParser<E> defaultValue(E value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public E parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(s -> Enum.valueOf(enumClass, s), defaultValue);
+ }
+
+ public E parseOptional() {
+ return parse(s -> Enum.valueOf(enumClass, s), null);
+ }
+ }
+
+ class DurationConfParser extends ConfParser<DurationConfParser, Duration> {
+ private Duration defaultValue;
+
+ @Override
+ protected DurationConfParser self() {
+ return this;
+ }
+
+ public DurationConfParser defaultValue(Duration value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public Duration parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(TimeUtils::parseDuration, defaultValue);
+ }
+
+ public Duration parseOptional() {
+ return parse(TimeUtils::parseDuration, null);
+ }
+ }
+
+ abstract class ConfParser<ThisT, T> {
+ private final List<String> optionNames = Lists.newArrayList();
+ private String tablePropertyName;
+ private ConfigOption<T> configOption;
+
+ protected abstract ThisT self();
+
+ public ThisT option(String name) {
+ this.optionNames.add(name);
+ return self();
+ }
+
+ public ThisT flinkConfig(ConfigOption<T> newConfigOption) {
+ this.configOption = newConfigOption;
+ return self();
+ }
+
+ public ThisT tableProperty(String name) {
+ this.tablePropertyName = name;
+ return self();
+ }
+
+ protected T parse(Function<String, T> conversion, T defaultValue) {
+ if (!optionNames.isEmpty()) {
+ for (String optionName : optionNames) {
+ String optionValue = options.get(optionName);
+ if (optionValue != null) {
+ return conversion.apply(optionValue);
+ }
+ }
+ }
+
+ if (configOption != null) {
+ T propertyValue = readableConfig.get(configOption);
+ if (propertyValue != null) {
+ return propertyValue;
+ }
+ }
+
+ if (tablePropertyName != null) {
+ String propertyValue = tableProperties.get(tablePropertyName);
+ if (propertyValue != null) {
+ return conversion.apply(propertyValue);
+ }
+ }
+
+ return defaultValue;
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java
new file mode 100644
index 0000000000..7c7afd24ed
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.description.Description;
+import org.apache.flink.configuration.description.TextElement;
+import org.apache.flink.table.api.TableEnvironment;
+import org.apache.iceberg.flink.source.assigner.SplitAssignerType;
+import org.apache.iceberg.util.ThreadPools;
+
+/**
+ * When constructing Flink Iceberg source via Java API, configs can be set in {@link Configuration}
+ * passed to source builder. E.g.
+ *
+ * <pre>
+ * configuration.setBoolean(FlinkConfigOptions.TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM, true);
+ * FlinkSource.forRowData()
+ * .flinkConf(configuration)
+ * ...
+ * </pre>
+ *
+ * <p>When using Flink SQL/table API, connector options can be set in Flink's {@link
+ * TableEnvironment}.
+ *
+ * <pre>
+ * TableEnvironment tEnv = createTableEnv();
+ * tEnv.getConfig()
+ * .getConfiguration()
+ * .setBoolean(FlinkConfigOptions.TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM, true);
+ * </pre>
+ */
+public class FlinkConfigOptions {
+
+ private FlinkConfigOptions() {}
+
+ public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM =
+ ConfigOptions.key("table.exec.iceberg.infer-source-parallelism")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "If is false, parallelism of source are set by config.\n"
+ + "If is true, source parallelism is inferred according to splits number.\n");
+
+ public static final ConfigOption<Integer> TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM_MAX =
+ ConfigOptions.key("table.exec.iceberg.infer-source-parallelism.max")
+ .intType()
+ .defaultValue(100)
+ .withDescription("Sets max infer parallelism for source operator.");
+
+ public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_EXPOSE_SPLIT_LOCALITY_INFO =
+ ConfigOptions.key("table.exec.iceberg.expose-split-locality-info")
+ .booleanType()
+ .noDefaultValue()
+ .withDescription(
+ "Expose split host information to use Flink's locality aware split assigner.");
+
+ public static final ConfigOption<Integer> SOURCE_READER_FETCH_BATCH_RECORD_COUNT =
+ ConfigOptions.key("table.exec.iceberg.fetch-batch-record-count")
+ .intType()
+ .defaultValue(2048)
+ .withDescription("The target number of records for Iceberg reader fetch batch.");
+
+ public static final ConfigOption<Integer> TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE =
+ ConfigOptions.key("table.exec.iceberg.worker-pool-size")
+ .intType()
+ .defaultValue(ThreadPools.WORKER_THREAD_POOL_SIZE)
+ .withDescription("The size of workers pool used to plan or scan manifests.");
+
+ public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_USE_FLIP27_SOURCE =
+ ConfigOptions.key("table.exec.iceberg.use-flip27-source")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Use the FLIP-27 based Iceberg source implementation.");
+
+ public static final ConfigOption<SplitAssignerType> TABLE_EXEC_SPLIT_ASSIGNER_TYPE =
+ ConfigOptions.key("table.exec.iceberg.split-assigner-type")
+ .enumType(SplitAssignerType.class)
+ .defaultValue(SplitAssignerType.SIMPLE)
+ .withDescription(
+ Description.builder()
+ .text("Split assigner type that determine how splits are assigned to readers.")
+ .linebreak()
+ .list(
+ TextElement.text(
+ SplitAssignerType.SIMPLE
+ + ": simple assigner that doesn't provide any guarantee on order or locality."))
+ .build());
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
new file mode 100644
index 0000000000..b7f1be4b93
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.Map;
+import java.util.Set;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.catalog.CatalogDatabaseImpl;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.utils.TableSchemaUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.flink.source.IcebergTableSource;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+
+public class FlinkDynamicTableFactory
+ implements DynamicTableSinkFactory, DynamicTableSourceFactory {
+ static final String FACTORY_IDENTIFIER = "iceberg";
+
+ private static final ConfigOption<String> CATALOG_NAME =
+ ConfigOptions.key("catalog-name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Catalog name");
+
+ private static final ConfigOption<String> CATALOG_TYPE =
+ ConfigOptions.key(FlinkCatalogFactory.ICEBERG_CATALOG_TYPE)
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Catalog type, the optional types are: custom, hadoop, hive.");
+
+ private static final ConfigOption<String> CATALOG_DATABASE =
+ ConfigOptions.key("catalog-database")
+ .stringType()
+ .defaultValue(FlinkCatalogFactory.DEFAULT_DATABASE_NAME)
+ .withDescription("Database name managed in the iceberg catalog.");
+
+ private static final ConfigOption<String> CATALOG_TABLE =
+ ConfigOptions.key("catalog-table")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Table name managed in the underlying iceberg catalog and database.");
+
+ private final FlinkCatalog catalog;
+
+ public FlinkDynamicTableFactory() {
+ this.catalog = null;
+ }
+
+ public FlinkDynamicTableFactory(FlinkCatalog catalog) {
+ this.catalog = catalog;
+ }
+
+ @Override
+ public DynamicTableSource createDynamicTableSource(Context context) {
+ ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
+ ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable();
+ Map<String, String> tableProps = resolvedCatalogTable.getOptions();
+ TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(resolvedCatalogTable.getSchema());
+
+ TableLoader tableLoader;
+ if (catalog != null) {
+ tableLoader = createTableLoader(catalog, objectIdentifier.toObjectPath());
+ } else {
+ tableLoader =
+ createTableLoader(
+ resolvedCatalogTable,
+ tableProps,
+ objectIdentifier.getDatabaseName(),
+ objectIdentifier.getObjectName());
+ }
+
+ return new IcebergTableSource(tableLoader, tableSchema, tableProps, context.getConfiguration());
+ }
+
+ @Override
+ public DynamicTableSink createDynamicTableSink(Context context) {
+ ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
+ ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable();
+ Map<String, String> writeProps = resolvedCatalogTable.getOptions();
+ TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(resolvedCatalogTable.getSchema());
+
+ TableLoader tableLoader;
+ if (catalog != null) {
+ tableLoader = createTableLoader(catalog, objectIdentifier.toObjectPath());
+ } else {
+ tableLoader =
+ createTableLoader(
+ resolvedCatalogTable,
+ writeProps,
+ objectIdentifier.getDatabaseName(),
+ objectIdentifier.getObjectName());
+ }
+
+ return new IcebergTableSink(tableLoader, tableSchema, context.getConfiguration(), writeProps);
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ Set<ConfigOption<?>> options = Sets.newHashSet();
+ options.add(CATALOG_TYPE);
+ options.add(CATALOG_NAME);
+ return options;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ Set<ConfigOption<?>> options = Sets.newHashSet();
+ options.add(CATALOG_DATABASE);
+ options.add(CATALOG_TABLE);
+ return options;
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return FACTORY_IDENTIFIER;
+ }
+
+ private static TableLoader createTableLoader(
+ ResolvedCatalogTable resolvedCatalogTable,
+ Map<String, String> tableProps,
+ String databaseName,
+ String tableName) {
+ Configuration flinkConf = new Configuration();
+ tableProps.forEach(flinkConf::setString);
+
+ String catalogName = flinkConf.getString(CATALOG_NAME);
+ Preconditions.checkNotNull(
+ catalogName, "Table property '%s' cannot be null", CATALOG_NAME.key());
+
+ String catalogDatabase = flinkConf.getString(CATALOG_DATABASE, databaseName);
+ Preconditions.checkNotNull(catalogDatabase, "The iceberg database name cannot be null");
+
+ String catalogTable = flinkConf.getString(CATALOG_TABLE, tableName);
+ Preconditions.checkNotNull(catalogTable, "The iceberg table name cannot be null");
+
+ org.apache.hadoop.conf.Configuration hadoopConf = FlinkCatalogFactory.clusterHadoopConf();
+ FlinkCatalogFactory factory = new FlinkCatalogFactory();
+ FlinkCatalog flinkCatalog =
+ (FlinkCatalog) factory.createCatalog(catalogName, tableProps, hadoopConf);
+ ObjectPath objectPath = new ObjectPath(catalogDatabase, catalogTable);
+
+ // Create database if not exists in the external catalog.
+ if (!flinkCatalog.databaseExists(catalogDatabase)) {
+ try {
+ flinkCatalog.createDatabase(
+ catalogDatabase, new CatalogDatabaseImpl(Maps.newHashMap(), null), true);
+ } catch (DatabaseAlreadyExistException e) {
+ throw new AlreadyExistsException(
+ e,
+ "Database %s already exists in the iceberg catalog %s.",
+ catalogName,
+ catalogDatabase);
+ }
+ }
+
+ // Create table if not exists in the external catalog.
+ if (!flinkCatalog.tableExists(objectPath)) {
+ try {
+ flinkCatalog.createIcebergTable(objectPath, resolvedCatalogTable, true);
+ } catch (TableAlreadyExistException e) {
+ throw new AlreadyExistsException(
+ e,
+ "Table %s already exists in the database %s and catalog %s",
+ catalogTable,
+ catalogDatabase,
+ catalogName);
+ }
+ }
+
+ return TableLoader.fromCatalog(
+ flinkCatalog.getCatalogLoader(), TableIdentifier.of(catalogDatabase, catalogTable));
+ }
+
+ private static TableLoader createTableLoader(FlinkCatalog catalog, ObjectPath objectPath) {
+ Preconditions.checkNotNull(catalog, "Flink catalog cannot be null");
+ return TableLoader.fromCatalog(catalog.getCatalogLoader(), catalog.toIdentifier(objectPath));
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java
new file mode 100644
index 0000000000..f35bb577fb
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.iceberg.EnvironmentContext;
+import org.apache.iceberg.flink.util.FlinkPackage;
+
+class FlinkEnvironmentContext {
+ private FlinkEnvironmentContext() {}
+
+ public static void init() {
+ EnvironmentContext.put(EnvironmentContext.ENGINE_NAME, "flink");
+ EnvironmentContext.put(EnvironmentContext.ENGINE_VERSION, FlinkPackage.version());
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java
new file mode 100644
index 0000000000..f2244d5137
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiFunction;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.flink.table.expressions.CallExpression;
+import org.apache.flink.table.expressions.FieldReferenceExpression;
+import org.apache.flink.table.expressions.ResolvedExpression;
+import org.apache.flink.table.expressions.ValueLiteralExpression;
+import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
+import org.apache.flink.table.functions.FunctionDefinition;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expression.Operation;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.util.DateTimeUtil;
+import org.apache.iceberg.util.NaNUtil;
+
+public class FlinkFilters {
+ private FlinkFilters() {}
+
+ private static final Pattern STARTS_WITH_PATTERN = Pattern.compile("([^%]+)%");
+
+ private static final Map<FunctionDefinition, Operation> FILTERS =
+ ImmutableMap.<FunctionDefinition, Operation>builder()
+ .put(BuiltInFunctionDefinitions.EQUALS, Operation.EQ)
+ .put(BuiltInFunctionDefinitions.NOT_EQUALS, Operation.NOT_EQ)
+ .put(BuiltInFunctionDefinitions.GREATER_THAN, Operation.GT)
+ .put(BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL, Operation.GT_EQ)
+ .put(BuiltInFunctionDefinitions.LESS_THAN, Operation.LT)
+ .put(BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL, Operation.LT_EQ)
+ .put(BuiltInFunctionDefinitions.IS_NULL, Operation.IS_NULL)
+ .put(BuiltInFunctionDefinitions.IS_NOT_NULL, Operation.NOT_NULL)
+ .put(BuiltInFunctionDefinitions.AND, Operation.AND)
+ .put(BuiltInFunctionDefinitions.OR, Operation.OR)
+ .put(BuiltInFunctionDefinitions.NOT, Operation.NOT)
+ .put(BuiltInFunctionDefinitions.LIKE, Operation.STARTS_WITH)
+ .buildOrThrow();
+
+ /**
+ * Convert flink expression to iceberg expression.
+ *
+ * <p>the BETWEEN, NOT_BETWEEN, IN expression will be converted by flink automatically. the
+ * BETWEEN will be converted to (GT_EQ AND LT_EQ), the NOT_BETWEEN will be converted to (LT_EQ OR
+ * GT_EQ), the IN will be converted to OR, so we do not add the conversion here
+ *
+ * @param flinkExpression the flink expression
+ * @return the iceberg expression
+ */
+ public static Optional<Expression> convert(
+ org.apache.flink.table.expressions.Expression flinkExpression) {
+ if (!(flinkExpression instanceof CallExpression)) {
+ return Optional.empty();
+ }
+
+ CallExpression call = (CallExpression) flinkExpression;
+ Operation op = FILTERS.get(call.getFunctionDefinition());
+ if (op != null) {
+ switch (op) {
+ case IS_NULL:
+ return onlyChildAs(call, FieldReferenceExpression.class)
+ .map(FieldReferenceExpression::getName)
+ .map(Expressions::isNull);
+
+ case NOT_NULL:
+ return onlyChildAs(call, FieldReferenceExpression.class)
+ .map(FieldReferenceExpression::getName)
+ .map(Expressions::notNull);
+
+ case LT:
+ return convertFieldAndLiteral(Expressions::lessThan, Expressions::greaterThan, call);
+
+ case LT_EQ:
+ return convertFieldAndLiteral(
+ Expressions::lessThanOrEqual, Expressions::greaterThanOrEqual, call);
+
+ case GT:
+ return convertFieldAndLiteral(Expressions::greaterThan, Expressions::lessThan, call);
+
+ case GT_EQ:
+ return convertFieldAndLiteral(
+ Expressions::greaterThanOrEqual, Expressions::lessThanOrEqual, call);
+
+ case EQ:
+ return convertFieldAndLiteral(
+ (ref, lit) -> {
+ if (NaNUtil.isNaN(lit)) {
+ return Expressions.isNaN(ref);
+ } else {
+ return Expressions.equal(ref, lit);
+ }
+ },
+ call);
+
+ case NOT_EQ:
+ return convertFieldAndLiteral(
+ (ref, lit) -> {
+ if (NaNUtil.isNaN(lit)) {
+ return Expressions.notNaN(ref);
+ } else {
+ return Expressions.notEqual(ref, lit);
+ }
+ },
+ call);
+
+ case NOT:
+ return onlyChildAs(call, CallExpression.class)
+ .flatMap(FlinkFilters::convert)
+ .map(Expressions::not);
+
+ case AND:
+ return convertLogicExpression(Expressions::and, call);
+
+ case OR:
+ return convertLogicExpression(Expressions::or, call);
+
+ case STARTS_WITH:
+ return convertLike(call);
+ }
+ }
+
+ return Optional.empty();
+ }
+
+ private static <T extends ResolvedExpression> Optional<T> onlyChildAs(
+ CallExpression call, Class<T> expectedChildClass) {
+ List<ResolvedExpression> children = call.getResolvedChildren();
+ if (children.size() != 1) {
+ return Optional.empty();
+ }
+
+ ResolvedExpression child = children.get(0);
+ if (!expectedChildClass.isInstance(child)) {
+ return Optional.empty();
+ }
+
+ return Optional.of(expectedChildClass.cast(child));
+ }
+
+ private static Optional<Expression> convertLike(CallExpression call) {
+ List<ResolvedExpression> args = call.getResolvedChildren();
+ if (args.size() != 2) {
+ return Optional.empty();
+ }
+
+ org.apache.flink.table.expressions.Expression left = args.get(0);
+ org.apache.flink.table.expressions.Expression right = args.get(1);
+
+ if (left instanceof FieldReferenceExpression && right instanceof ValueLiteralExpression) {
+ String name = ((FieldReferenceExpression) left).getName();
+ return convertLiteral((ValueLiteralExpression) right)
+ .flatMap(
+ lit -> {
+ if (lit instanceof String) {
+ String pattern = (String) lit;
+ Matcher matcher = STARTS_WITH_PATTERN.matcher(pattern);
+ // exclude special char of LIKE
+ // '_' is the wildcard of the SQL LIKE
+ if (!pattern.contains("_") && matcher.matches()) {
+ return Optional.of(Expressions.startsWith(name, matcher.group(1)));
+ }
+ }
+
+ return Optional.empty();
+ });
+ }
+
+ return Optional.empty();
+ }
+
+ private static Optional<Expression> convertLogicExpression(
+ BiFunction<Expression, Expression, Expression> function, CallExpression call) {
+ List<ResolvedExpression> args = call.getResolvedChildren();
+ if (args == null || args.size() != 2) {
+ return Optional.empty();
+ }
+
+ Optional<Expression> left = convert(args.get(0));
+ Optional<Expression> right = convert(args.get(1));
+ if (left.isPresent() && right.isPresent()) {
+ return Optional.of(function.apply(left.get(), right.get()));
+ }
+
+ return Optional.empty();
+ }
+
+ private static Optional<Object> convertLiteral(ValueLiteralExpression expression) {
+ Optional<?> value =
+ expression.getValueAs(
+ expression.getOutputDataType().getLogicalType().getDefaultConversion());
+ return value.map(
+ o -> {
+ if (o instanceof LocalDateTime) {
+ return DateTimeUtil.microsFromTimestamp((LocalDateTime) o);
+ } else if (o instanceof Instant) {
+ return DateTimeUtil.microsFromInstant((Instant) o);
+ } else if (o instanceof LocalTime) {
+ return DateTimeUtil.microsFromTime((LocalTime) o);
+ } else if (o instanceof LocalDate) {
+ return DateTimeUtil.daysFromDate((LocalDate) o);
+ }
+
+ return o;
+ });
+ }
+
+ private static Optional<Expression> convertFieldAndLiteral(
+ BiFunction<String, Object, Expression> expr, CallExpression call) {
+ return convertFieldAndLiteral(expr, expr, call);
+ }
+
+ private static Optional<Expression> convertFieldAndLiteral(
+ BiFunction<String, Object, Expression> convertLR,
+ BiFunction<String, Object, Expression> convertRL,
+ CallExpression call) {
+ List<ResolvedExpression> args = call.getResolvedChildren();
+ if (args.size() != 2) {
+ return Optional.empty();
+ }
+
+ org.apache.flink.table.expressions.Expression left = args.get(0);
+ org.apache.flink.table.expressions.Expression right = args.get(1);
+
+ if (left instanceof FieldReferenceExpression && right instanceof ValueLiteralExpression) {
+ String name = ((FieldReferenceExpression) left).getName();
+ Optional<Object> lit = convertLiteral((ValueLiteralExpression) right);
+ if (lit.isPresent()) {
+ return Optional.of(convertLR.apply(name, lit.get()));
+ }
+ } else if (left instanceof ValueLiteralExpression
+ && right instanceof FieldReferenceExpression) {
+ Optional<Object> lit = convertLiteral((ValueLiteralExpression) left);
+ String name = ((FieldReferenceExpression) right).getName();
+ if (lit.isPresent()) {
+ return Optional.of(convertRL.apply(name, lit.get()));
+ }
+ }
+
+ return Optional.empty();
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java
new file mode 100644
index 0000000000..767d4497ac
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.types.FixupTypes;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+/**
+ * The uuid and fixed are converted to the same Flink type. Conversion back can produce only one,
+ * which may not be correct.
+ */
+class FlinkFixupTypes extends FixupTypes {
+
+ private FlinkFixupTypes(Schema referenceSchema) {
+ super(referenceSchema);
+ }
+
+ static Schema fixup(Schema schema, Schema referenceSchema) {
+ return new Schema(
+ TypeUtil.visit(schema, new FlinkFixupTypes(referenceSchema)).asStructType().fields());
+ }
+
+ @Override
+ protected boolean fixupPrimitive(Type.PrimitiveType type, Type source) {
+ if (type instanceof Types.FixedType) {
+ int length = ((Types.FixedType) type).length();
+ return source.typeId() == Type.TypeID.UUID && length == 16;
+ }
+ return false;
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java
new file mode 100644
index 0000000000..804a956ec9
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.util.TimeUtils;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.flink.source.StreamingStartingStrategy;
+
+public class FlinkReadConf {
+
+ private final FlinkConfParser confParser;
+
+ public FlinkReadConf(
+ Table table, Map<String, String> readOptions, ReadableConfig readableConfig) {
+ this.confParser = new FlinkConfParser(table, readOptions, readableConfig);
+ }
+
+ public Long snapshotId() {
+ return confParser.longConf().option(FlinkReadOptions.SNAPSHOT_ID.key()).parseOptional();
+ }
+
+ public String tag() {
+ return confParser.stringConf().option(FlinkReadOptions.TAG.key()).parseOptional();
+ }
+
+ public String startTag() {
+ return confParser.stringConf().option(FlinkReadOptions.START_TAG.key()).parseOptional();
+ }
+
+ public String endTag() {
+ return confParser.stringConf().option(FlinkReadOptions.END_TAG.key()).parseOptional();
+ }
+
+ public String branch() {
+ return confParser.stringConf().option(FlinkReadOptions.BRANCH.key()).parseOptional();
+ }
+
+ public boolean caseSensitive() {
+ return confParser
+ .booleanConf()
+ .option(FlinkReadOptions.CASE_SENSITIVE)
+ .flinkConfig(FlinkReadOptions.CASE_SENSITIVE_OPTION)
+ .defaultValue(FlinkReadOptions.CASE_SENSITIVE_OPTION.defaultValue())
+ .parse();
+ }
+
+ public Long asOfTimestamp() {
+ return confParser.longConf().option(FlinkReadOptions.AS_OF_TIMESTAMP.key()).parseOptional();
+ }
+
+ public StreamingStartingStrategy startingStrategy() {
+ return confParser
+ .enumConfParser(StreamingStartingStrategy.class)
+ .option(FlinkReadOptions.STARTING_STRATEGY)
+ .flinkConfig(FlinkReadOptions.STARTING_STRATEGY_OPTION)
+ .defaultValue(StreamingStartingStrategy.INCREMENTAL_FROM_LATEST_SNAPSHOT)
+ .parse();
+ }
+
+ public Long startSnapshotTimestamp() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.START_SNAPSHOT_TIMESTAMP.key())
+ .parseOptional();
+ }
+
+ public Long startSnapshotId() {
+ return confParser.longConf().option(FlinkReadOptions.START_SNAPSHOT_ID.key()).parseOptional();
+ }
+
+ public Long endSnapshotId() {
+ return confParser.longConf().option(FlinkReadOptions.END_SNAPSHOT_ID.key()).parseOptional();
+ }
+
+ public long splitSize() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.SPLIT_SIZE)
+ .flinkConfig(FlinkReadOptions.SPLIT_SIZE_OPTION)
+ .tableProperty(TableProperties.SPLIT_SIZE)
+ .defaultValue(TableProperties.SPLIT_SIZE_DEFAULT)
+ .parse();
+ }
+
+ public int splitLookback() {
+ return confParser
+ .intConf()
+ .option(FlinkReadOptions.SPLIT_LOOKBACK)
+ .flinkConfig(FlinkReadOptions.SPLIT_LOOKBACK_OPTION)
+ .tableProperty(TableProperties.SPLIT_LOOKBACK)
+ .defaultValue(TableProperties.SPLIT_LOOKBACK_DEFAULT)
+ .parse();
+ }
+
+ public long splitFileOpenCost() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.SPLIT_FILE_OPEN_COST)
+ .flinkConfig(FlinkReadOptions.SPLIT_FILE_OPEN_COST_OPTION)
+ .tableProperty(TableProperties.SPLIT_OPEN_FILE_COST)
+ .defaultValue(TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT)
+ .parse();
+ }
+
+ public boolean streaming() {
+ return confParser
+ .booleanConf()
+ .option(FlinkReadOptions.STREAMING)
+ .flinkConfig(FlinkReadOptions.STREAMING_OPTION)
+ .defaultValue(FlinkReadOptions.STREAMING_OPTION.defaultValue())
+ .parse();
+ }
+
+ public Duration monitorInterval() {
+ String duration =
+ confParser
+ .stringConf()
+ .option(FlinkReadOptions.MONITOR_INTERVAL)
+ .flinkConfig(FlinkReadOptions.MONITOR_INTERVAL_OPTION)
+ .defaultValue(FlinkReadOptions.MONITOR_INTERVAL_OPTION.defaultValue())
+ .parse();
+
+ return TimeUtils.parseDuration(duration);
+ }
+
+ public boolean includeColumnStats() {
+ return confParser
+ .booleanConf()
+ .option(FlinkReadOptions.INCLUDE_COLUMN_STATS)
+ .flinkConfig(FlinkReadOptions.INCLUDE_COLUMN_STATS_OPTION)
+ .defaultValue(FlinkReadOptions.INCLUDE_COLUMN_STATS_OPTION.defaultValue())
+ .parse();
+ }
+
+ public int maxPlanningSnapshotCount() {
+ return confParser
+ .intConf()
+ .option(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT)
+ .flinkConfig(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT_OPTION)
+ .defaultValue(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT_OPTION.defaultValue())
+ .parse();
+ }
+
+ public String nameMapping() {
+ return confParser.stringConf().option(TableProperties.DEFAULT_NAME_MAPPING).parseOptional();
+ }
+
+ public long limit() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.LIMIT)
+ .flinkConfig(FlinkReadOptions.LIMIT_OPTION)
+ .defaultValue(FlinkReadOptions.LIMIT_OPTION.defaultValue())
+ .parse();
+ }
+
+ public int workerPoolSize() {
+ return confParser
+ .intConf()
+ .option(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.key())
+ .flinkConfig(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE)
+ .defaultValue(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.defaultValue())
+ .parse();
+ }
+
+ public int maxAllowedPlanningFailures() {
+ return confParser
+ .intConf()
+ .option(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES)
+ .flinkConfig(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES_OPTION)
+ .defaultValue(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES_OPTION.defaultValue())
+ .parse();
+ }
+
+ public String watermarkColumn() {
+ return confParser
+ .stringConf()
+ .option(FlinkReadOptions.WATERMARK_COLUMN)
+ .flinkConfig(FlinkReadOptions.WATERMARK_COLUMN_OPTION)
+ .defaultValue(FlinkReadOptions.WATERMARK_COLUMN_OPTION.defaultValue())
+ .parseOptional();
+ }
+
+ public TimeUnit watermarkColumnTimeUnit() {
+ return confParser
+ .enumConfParser(TimeUnit.class)
+ .option(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT)
+ .flinkConfig(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT_OPTION)
+ .defaultValue(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT_OPTION.defaultValue())
+ .parse();
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java
new file mode 100644
index 0000000000..1bbd88146c
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.flink.source.StreamingStartingStrategy;
+
+/** Flink source read options */
+public class FlinkReadOptions {
+ private static final String PREFIX = "connector.iceberg.";
+
+ private FlinkReadOptions() {}
+
+ public static final ConfigOption<Long> SNAPSHOT_ID =
+ ConfigOptions.key("snapshot-id").longType().defaultValue(null);
+
+ public static final ConfigOption<String> TAG =
+ ConfigOptions.key("tag").stringType().defaultValue(null);
+
+ public static final ConfigOption<String> BRANCH =
+ ConfigOptions.key("branch").stringType().defaultValue(null);
+
+ public static final ConfigOption<String> START_TAG =
+ ConfigOptions.key("start-tag").stringType().defaultValue(null);
+
+ public static final ConfigOption<String> END_TAG =
+ ConfigOptions.key("end-tag").stringType().defaultValue(null);
+
+ public static final String CASE_SENSITIVE = "case-sensitive";
+ public static final ConfigOption<Boolean> CASE_SENSITIVE_OPTION =
+ ConfigOptions.key(PREFIX + CASE_SENSITIVE).booleanType().defaultValue(false);
+
+ public static final ConfigOption<Long> AS_OF_TIMESTAMP =
+ ConfigOptions.key("as-of-timestamp").longType().defaultValue(null);
+
+ public static final String STARTING_STRATEGY = "starting-strategy";
+ public static final ConfigOption<StreamingStartingStrategy> STARTING_STRATEGY_OPTION =
+ ConfigOptions.key(PREFIX + STARTING_STRATEGY)
+ .enumType(StreamingStartingStrategy.class)
+ .defaultValue(StreamingStartingStrategy.INCREMENTAL_FROM_LATEST_SNAPSHOT);
+
+ public static final ConfigOption<Long> START_SNAPSHOT_TIMESTAMP =
+ ConfigOptions.key("start-snapshot-timestamp").longType().defaultValue(null);
+
+ public static final ConfigOption<Long> START_SNAPSHOT_ID =
+ ConfigOptions.key("start-snapshot-id").longType().defaultValue(null);
+
+ public static final ConfigOption<Long> END_SNAPSHOT_ID =
+ ConfigOptions.key("end-snapshot-id").longType().defaultValue(null);
+
+ public static final String SPLIT_SIZE = "split-size";
+ public static final ConfigOption<Long> SPLIT_SIZE_OPTION =
+ ConfigOptions.key(PREFIX + SPLIT_SIZE)
+ .longType()
+ .defaultValue(TableProperties.SPLIT_SIZE_DEFAULT);
+
+ public static final String SPLIT_LOOKBACK = "split-lookback";
+ public static final ConfigOption<Integer> SPLIT_LOOKBACK_OPTION =
+ ConfigOptions.key(PREFIX + SPLIT_LOOKBACK)
+ .intType()
+ .defaultValue(TableProperties.SPLIT_LOOKBACK_DEFAULT);
+
+ public static final String SPLIT_FILE_OPEN_COST = "split-file-open-cost";
+ public static final ConfigOption<Long> SPLIT_FILE_OPEN_COST_OPTION =
+ ConfigOptions.key(PREFIX + SPLIT_FILE_OPEN_COST)
+ .longType()
+ .defaultValue(TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT);
+
+ public static final String STREAMING = "streaming";
+ public static final ConfigOption<Boolean> STREAMING_OPTION =
+ ConfigOptions.key(PREFIX + STREAMING).booleanType().defaultValue(false);
+
+ public static final String MONITOR_INTERVAL = "monitor-interval";
+ public static final ConfigOption<String> MONITOR_INTERVAL_OPTION =
+ ConfigOptions.key(PREFIX + MONITOR_INTERVAL).stringType().defaultValue("60s");
+
+ public static final String INCLUDE_COLUMN_STATS = "include-column-stats";
+ public static final ConfigOption<Boolean> INCLUDE_COLUMN_STATS_OPTION =
+ ConfigOptions.key(PREFIX + INCLUDE_COLUMN_STATS).booleanType().defaultValue(false);
+
+ public static final String MAX_PLANNING_SNAPSHOT_COUNT = "max-planning-snapshot-count";
+ public static final ConfigOption<Integer> MAX_PLANNING_SNAPSHOT_COUNT_OPTION =
+ ConfigOptions.key(PREFIX + MAX_PLANNING_SNAPSHOT_COUNT)
+ .intType()
+ .defaultValue(Integer.MAX_VALUE);
+
+ public static final String LIMIT = "limit";
+ public static final ConfigOption<Long> LIMIT_OPTION =
+ ConfigOptions.key(PREFIX + LIMIT).longType().defaultValue(-1L);
+
+ public static final String MAX_ALLOWED_PLANNING_FAILURES = "max-allowed-planning-failures";
+ public static final ConfigOption<Integer> MAX_ALLOWED_PLANNING_FAILURES_OPTION =
+ ConfigOptions.key(PREFIX + MAX_ALLOWED_PLANNING_FAILURES).intType().defaultValue(3);
+
+ public static final String WATERMARK_COLUMN = "watermark-column";
+ public static final ConfigOption<String> WATERMARK_COLUMN_OPTION =
+ ConfigOptions.key(PREFIX + WATERMARK_COLUMN).stringType().noDefaultValue();
+
+ public static final String WATERMARK_COLUMN_TIME_UNIT = "watermark-column-time-unit";
+ public static final ConfigOption<TimeUnit> WATERMARK_COLUMN_TIME_UNIT_OPTION =
+ ConfigOptions.key(PREFIX + WATERMARK_COLUMN_TIME_UNIT)
+ .enumType(TimeUnit.class)
+ .defaultValue(TimeUnit.MICROSECONDS);
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
new file mode 100644
index 0000000000..4790dc85bf
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.utils.TypeConversions;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+/**
+ * Converter between Flink types and Iceberg type. The conversion is not a 1:1 mapping that not
+ * allows back-and-forth conversion. So some information might get lost during the back-and-forth
+ * conversion.
+ *
+ * <p>This inconsistent types:
+ *
+ * <ul>
+ * <li>map Iceberg UUID type to Flink BinaryType(16)
+ * <li>map Flink VarCharType(_) and CharType(_) to Iceberg String type
+ * <li>map Flink VarBinaryType(_) to Iceberg Binary type
+ * <li>map Flink TimeType(_) to Iceberg Time type (microseconds)
+ * <li>map Flink TimestampType(_) to Iceberg Timestamp without zone type (microseconds)
+ * <li>map Flink LocalZonedTimestampType(_) to Iceberg Timestamp with zone type (microseconds)
+ * <li>map Flink MultiSetType to Iceberg Map type(element, int)
+ * </ul>
+ *
+ * <p>
+ */
+public class FlinkSchemaUtil {
+
+ private FlinkSchemaUtil() {}
+
+ /** @deprecated Use {@link #convert(ResolvedSchema)} instead. */
+ @Deprecated
+ public static Schema convert(TableSchema schema) {
+ LogicalType schemaType = schema.toRowDataType().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be row type.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ if (schema.getPrimaryKey().isPresent()) {
+ return freshIdentifierFieldIds(icebergSchema, schema.getPrimaryKey().get().getColumns());
+ } else {
+ return icebergSchema;
+ }
+ }
+
+ /** Convert the flink table schema to apache iceberg schema with column comment. */
+ public static Schema convert(ResolvedSchema flinkSchema) {
+ List<Column> tableColumns = flinkSchema.getColumns();
+ // copy from org.apache.flink.table.api.Schema#toRowDataType
+ DataTypes.Field[] fields =
+ tableColumns.stream()
+ .map(
+ column -> {
+ if (column.getComment().isPresent()) {
+ return DataTypes.FIELD(
+ column.getName(), column.getDataType(), column.getComment().get());
+ } else {
+ return DataTypes.FIELD(column.getName(), column.getDataType());
+ }
+ })
+ .toArray(DataTypes.Field[]::new);
+
+ LogicalType schemaType = DataTypes.ROW(fields).notNull().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be row type.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ if (flinkSchema.getPrimaryKey().isPresent()) {
+ return freshIdentifierFieldIds(icebergSchema, flinkSchema.getPrimaryKey().get().getColumns());
+ } else {
+ return icebergSchema;
+ }
+ }
+
+ private static Schema freshIdentifierFieldIds(Schema icebergSchema, List<String> primaryKeys) {
+ // Locate the identifier field id list.
+ Set<Integer> identifierFieldIds = Sets.newHashSet();
+ for (String primaryKey : primaryKeys) {
+ Types.NestedField field = icebergSchema.findField(primaryKey);
+ Preconditions.checkNotNull(
+ field,
+ "Cannot find field ID for the primary key column %s in schema %s",
+ primaryKey,
+ icebergSchema);
+ identifierFieldIds.add(field.fieldId());
+ }
+ return new Schema(
+ icebergSchema.schemaId(), icebergSchema.asStruct().fields(), identifierFieldIds);
+ }
+
+ /**
+ * Convert a Flink {@link TableSchema} to a {@link Schema} based on the given schema.
+ *
+ * <p>This conversion does not assign new ids; it uses ids from the base schema.
+ *
+ * <p>Data types, field order, and nullability will match the Flink type. This conversion may
+ * return a schema that is not compatible with base schema.
+ *
+ * @param baseSchema a Schema on which conversion is based
+ * @param flinkSchema a Flink TableSchema
+ * @return the equivalent Schema
+ * @throws IllegalArgumentException if the type cannot be converted or there are missing ids
+ */
+ public static Schema convert(Schema baseSchema, TableSchema flinkSchema) {
+ // convert to a type with fresh ids
+ Types.StructType struct = convert(flinkSchema).asStruct();
+ // reassign ids to match the base schema
+ Schema schema = TypeUtil.reassignIds(new Schema(struct.fields()), baseSchema);
+ // reassign doc to match the base schema
+ schema = TypeUtil.reassignDoc(schema, baseSchema);
+
+ // fix types that can't be represented in Flink (UUID)
+ Schema fixedSchema = FlinkFixupTypes.fixup(schema, baseSchema);
+ if (flinkSchema.getPrimaryKey().isPresent()) {
+ return freshIdentifierFieldIds(fixedSchema, flinkSchema.getPrimaryKey().get().getColumns());
+ } else {
+ return fixedSchema;
+ }
+ }
+
+ /**
+ * Convert a {@link Schema} to a {@link RowType Flink type}.
+ *
+ * @param schema a Schema
+ * @return the equivalent Flink type
+ * @throws IllegalArgumentException if the type cannot be converted to Flink
+ */
+ public static RowType convert(Schema schema) {
+ return (RowType) TypeUtil.visit(schema, new TypeToFlinkType());
+ }
+
+ /**
+ * Convert a {@link Type} to a {@link LogicalType Flink type}.
+ *
+ * @param type a Type
+ * @return the equivalent Flink type
+ * @throws IllegalArgumentException if the type cannot be converted to Flink
+ */
+ public static LogicalType convert(Type type) {
+ return TypeUtil.visit(type, new TypeToFlinkType());
+ }
+
+ /**
+ * Convert a {@link LogicalType Flink type} to a {@link Type}.
+ *
+ * @param flinkType a FlinkType
+ * @return the equivalent Iceberg type
+ */
+ public static Type convert(LogicalType flinkType) {
+ return flinkType.accept(new FlinkTypeToType());
+ }
+
+ /**
+ * Convert a {@link RowType} to a {@link TableSchema}.
+ *
+ * @param rowType a RowType
+ * @return Flink TableSchema
+ */
+ public static TableSchema toSchema(RowType rowType) {
+ TableSchema.Builder builder = TableSchema.builder();
+ for (RowType.RowField field : rowType.getFields()) {
+ builder.field(field.getName(), TypeConversions.fromLogicalToDataType(field.getType()));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Convert a {@link Schema} to a {@link TableSchema}.
+ *
+ * @param schema iceberg schema to convert.
+ * @return Flink TableSchema.
+ */
+ public static TableSchema toSchema(Schema schema) {
+ TableSchema.Builder builder = TableSchema.builder();
+
+ // Add columns.
+ for (RowType.RowField field : convert(schema).getFields()) {
+ builder.field(field.getName(), TypeConversions.fromLogicalToDataType(field.getType()));
+ }
+
+ // Add primary key.
+ Set<Integer> identifierFieldIds = schema.identifierFieldIds();
+ if (!identifierFieldIds.isEmpty()) {
+ List<String> columns = Lists.newArrayListWithExpectedSize(identifierFieldIds.size());
+ for (Integer identifierFieldId : identifierFieldIds) {
+ String columnName = schema.findColumnName(identifierFieldId);
+ Preconditions.checkNotNull(
+ columnName, "Cannot find field with id %s in schema %s", identifierFieldId, schema);
+
+ columns.add(columnName);
+ }
+ builder.primaryKey(columns.toArray(new String[0]));
+ }
+
+ return builder.build();
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java
new file mode 100644
index 0000000000..5fbd84909d
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.expressions.Evaluator;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.types.Types;
+
+public class FlinkSourceFilter implements FilterFunction<RowData> {
+
+ private final RowType rowType;
+ private final Evaluator evaluator;
+ private final Types.StructType struct;
+ private volatile RowDataWrapper wrapper;
+
+ public FlinkSourceFilter(Schema schema, Expression expr, boolean caseSensitive) {
+ this.rowType = FlinkSchemaUtil.convert(schema);
+ this.struct = schema.asStruct();
+ this.evaluator = new Evaluator(struct, expr, caseSensitive);
+ }
+
+ @Override
+ public boolean filter(RowData value) {
+ if (wrapper == null) {
+ this.wrapper = new RowDataWrapper(rowType, struct);
+ }
+ return evaluator.eval(wrapper.wrap(value));
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java
new file mode 100644
index 0000000000..408065f060
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.BigIntType;
+import org.apache.flink.table.types.logical.BinaryType;
+import org.apache.flink.table.types.logical.BooleanType;
+import org.apache.flink.table.types.logical.CharType;
+import org.apache.flink.table.types.logical.DateType;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.DoubleType;
+import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.MultisetType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.SmallIntType;
+import org.apache.flink.table.types.logical.TimeType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.table.types.logical.TinyIntType;
+import org.apache.flink.table.types.logical.VarBinaryType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+class FlinkTypeToType extends FlinkTypeVisitor<Type> {
+
+ private final RowType root;
+ private int nextId;
+
+ FlinkTypeToType() {
+ this.root = null;
+ }
+
+ FlinkTypeToType(RowType root) {
+ this.root = root;
+ // the root struct's fields use the first ids
+ this.nextId = root.getFieldCount();
+ }
+
+ private int getNextId() {
+ int next = nextId;
+ nextId += 1;
+ return next;
+ }
+
+ @Override
+ public Type visit(CharType charType) {
+ return Types.StringType.get();
+ }
+
+ @Override
+ public Type visit(VarCharType varCharType) {
+ return Types.StringType.get();
+ }
+
+ @Override
+ public Type visit(BooleanType booleanType) {
+ return Types.BooleanType.get();
+ }
+
+ @Override
+ public Type visit(BinaryType binaryType) {
+ return Types.FixedType.ofLength(binaryType.getLength());
+ }
+
+ @Override
+ public Type visit(VarBinaryType varBinaryType) {
+ return Types.BinaryType.get();
+ }
+
+ @Override
+ public Type visit(DecimalType decimalType) {
+ return Types.DecimalType.of(decimalType.getPrecision(), decimalType.getScale());
+ }
+
+ @Override
+ public Type visit(TinyIntType tinyIntType) {
+ return Types.IntegerType.get();
+ }
+
+ @Override
+ public Type visit(SmallIntType smallIntType) {
+ return Types.IntegerType.get();
+ }
+
+ @Override
+ public Type visit(IntType intType) {
+ return Types.IntegerType.get();
+ }
+
+ @Override
+ public Type visit(BigIntType bigIntType) {
+ return Types.LongType.get();
+ }
+
+ @Override
+ public Type visit(FloatType floatType) {
+ return Types.FloatType.get();
+ }
+
+ @Override
+ public Type visit(DoubleType doubleType) {
+ return Types.DoubleType.get();
+ }
+
+ @Override
+ public Type visit(DateType dateType) {
+ return Types.DateType.get();
+ }
+
+ @Override
+ public Type visit(TimeType timeType) {
+ return Types.TimeType.get();
+ }
+
+ @Override
+ public Type visit(TimestampType timestampType) {
+ return Types.TimestampType.withoutZone();
+ }
+
+ @Override
+ public Type visit(LocalZonedTimestampType localZonedTimestampType) {
+ return Types.TimestampType.withZone();
+ }
+
+ @Override
+ public Type visit(ArrayType arrayType) {
+ Type elementType = arrayType.getElementType().accept(this);
+ if (arrayType.getElementType().isNullable()) {
+ return Types.ListType.ofOptional(getNextId(), elementType);
+ } else {
+ return Types.ListType.ofRequired(getNextId(), elementType);
+ }
+ }
+
+ @Override
+ public Type visit(MultisetType multisetType) {
+ Type elementType = multisetType.getElementType().accept(this);
+ return Types.MapType.ofRequired(getNextId(), getNextId(), elementType, Types.IntegerType.get());
+ }
+
+ @Override
+ public Type visit(MapType mapType) {
+ // keys in map are not allowed to be null.
+ Type keyType = mapType.getKeyType().accept(this);
+ Type valueType = mapType.getValueType().accept(this);
+ if (mapType.getValueType().isNullable()) {
+ return Types.MapType.ofOptional(getNextId(), getNextId(), keyType, valueType);
+ } else {
+ return Types.MapType.ofRequired(getNextId(), getNextId(), keyType, valueType);
+ }
+ }
+
+ @Override
+ @SuppressWarnings("ReferenceEquality")
+ public Type visit(RowType rowType) {
+ List<Types.NestedField> newFields = Lists.newArrayListWithExpectedSize(rowType.getFieldCount());
+ boolean isRoot = root == rowType;
+
+ List<Type> types =
+ rowType.getFields().stream()
+ .map(f -> f.getType().accept(this))
+ .collect(Collectors.toList());
+
+ for (int i = 0; i < rowType.getFieldCount(); i++) {
+ int id = isRoot ? i : getNextId();
+
+ RowType.RowField field = rowType.getFields().get(i);
+ String name = field.getName();
+ String comment = field.getDescription().orElse(null);
+
+ if (field.getType().isNullable()) {
+ newFields.add(Types.NestedField.optional(id, name, types.get(i), comment));
+ } else {
+ newFields.add(Types.NestedField.required(id, name, types.get(i), comment));
+ }
+ }
+
+ return Types.StructType.of(newFields);
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java
new file mode 100644
index 0000000000..f3de241608
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.table.types.logical.DayTimeIntervalType;
+import org.apache.flink.table.types.logical.DistinctType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeVisitor;
+import org.apache.flink.table.types.logical.NullType;
+import org.apache.flink.table.types.logical.RawType;
+import org.apache.flink.table.types.logical.StructuredType;
+import org.apache.flink.table.types.logical.SymbolType;
+import org.apache.flink.table.types.logical.YearMonthIntervalType;
+import org.apache.flink.table.types.logical.ZonedTimestampType;
+
+public abstract class FlinkTypeVisitor<T> implements LogicalTypeVisitor<T> {
+
+ // ------------------------- Unsupported types ------------------------------
+
+ @Override
+ public T visit(ZonedTimestampType zonedTimestampType) {
+ throw new UnsupportedOperationException("Unsupported ZonedTimestampType.");
+ }
+
+ @Override
+ public T visit(YearMonthIntervalType yearMonthIntervalType) {
+ throw new UnsupportedOperationException("Unsupported YearMonthIntervalType.");
+ }
+
+ @Override
+ public T visit(DayTimeIntervalType dayTimeIntervalType) {
+ throw new UnsupportedOperationException("Unsupported DayTimeIntervalType.");
+ }
+
+ @Override
+ public T visit(DistinctType distinctType) {
+ throw new UnsupportedOperationException("Unsupported DistinctType.");
+ }
+
+ @Override
+ public T visit(StructuredType structuredType) {
+ throw new UnsupportedOperationException("Unsupported StructuredType.");
+ }
+
+ @Override
+ public T visit(NullType nullType) {
+ throw new UnsupportedOperationException("Unsupported NullType.");
+ }
+
+ @Override
+ public T visit(RawType<?> rawType) {
+ throw new UnsupportedOperationException("Unsupported RawType.");
+ }
+
+ @Override
+ public T visit(SymbolType<?> symbolType) {
+ throw new UnsupportedOperationException("Unsupported SymbolType.");
+ }
+
+ @Override
+ public T visit(LogicalType other) {
+ throw new UnsupportedOperationException("Unsupported type: " + other);
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java
new file mode 100644
index 0000000000..ca7b1120bc
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import java.util.Map;
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.iceberg.DistributionMode;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+
+/**
+ * A class for common Iceberg configs for Flink writes.
+ *
+ * <p>If a config is set at multiple levels, the following order of precedence is used (top to
+ * bottom):
+ *
+ * <ol>
+ * <li>Write options
+ * <li>flink ReadableConfig
+ * <li>Table metadata
+ * </ol>
+ *
+ * The most specific value is set in write options and takes precedence over all other configs. If
+ * no write option is provided, this class checks the flink configuration for any overrides. If no
+ * applicable value is found in the write options, this class uses the table metadata.
+ *
+ * <p>Note this class is NOT meant to be serialized.
+ */
+public class FlinkWriteConf {
+
+ private final FlinkConfParser confParser;
+
+ public FlinkWriteConf(
+ Table table, Map<String, String> writeOptions, ReadableConfig readableConfig) {
+ this.confParser = new FlinkConfParser(table, writeOptions, readableConfig);
+ }
+
+ public boolean overwriteMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.OVERWRITE_MODE.key())
+ .flinkConfig(FlinkWriteOptions.OVERWRITE_MODE)
+ .defaultValue(FlinkWriteOptions.OVERWRITE_MODE.defaultValue())
+ .parse();
+ }
+
+ public boolean upsertMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.WRITE_UPSERT_ENABLED.key())
+ .flinkConfig(FlinkWriteOptions.WRITE_UPSERT_ENABLED)
+ .tableProperty(TableProperties.UPSERT_ENABLED)
+ .defaultValue(TableProperties.UPSERT_ENABLED_DEFAULT)
+ .parse();
+ }
+
+ public FileFormat dataFileFormat() {
+ String valueAsString =
+ confParser
+ .stringConf()
+ .option(FlinkWriteOptions.WRITE_FORMAT.key())
+ .flinkConfig(FlinkWriteOptions.WRITE_FORMAT)
+ .tableProperty(TableProperties.DEFAULT_FILE_FORMAT)
+ .defaultValue(TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)
+ .parse();
+ return FileFormat.fromString(valueAsString);
+ }
+
+ public long targetDataFileSize() {
+ return confParser
+ .longConf()
+ .option(FlinkWriteOptions.TARGET_FILE_SIZE_BYTES.key())
+ .flinkConfig(FlinkWriteOptions.TARGET_FILE_SIZE_BYTES)
+ .tableProperty(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES)
+ .defaultValue(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT)
+ .parse();
+ }
+
+ public String parquetCompressionCodec() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
+ .tableProperty(TableProperties.PARQUET_COMPRESSION)
+ .defaultValue(TableProperties.PARQUET_COMPRESSION_DEFAULT)
+ .parse();
+ }
+
+ public String parquetCompressionLevel() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_LEVEL.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_LEVEL)
+ .tableProperty(TableProperties.PARQUET_COMPRESSION_LEVEL)
+ .defaultValue(TableProperties.PARQUET_COMPRESSION_LEVEL_DEFAULT)
+ .parseOptional();
+ }
+
+ public String avroCompressionCodec() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
+ .tableProperty(TableProperties.AVRO_COMPRESSION)
+ .defaultValue(TableProperties.AVRO_COMPRESSION_DEFAULT)
+ .parse();
+ }
+
+ public String avroCompressionLevel() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_LEVEL.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_LEVEL)
+ .tableProperty(TableProperties.AVRO_COMPRESSION_LEVEL)
+ .defaultValue(TableProperties.AVRO_COMPRESSION_LEVEL_DEFAULT)
+ .parseOptional();
+ }
+
+ public String orcCompressionCodec() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
+ .tableProperty(TableProperties.ORC_COMPRESSION)
+ .defaultValue(TableProperties.ORC_COMPRESSION_DEFAULT)
+ .parse();
+ }
+
+ public String orcCompressionStrategy() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_STRATEGY.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_STRATEGY)
+ .tableProperty(TableProperties.ORC_COMPRESSION_STRATEGY)
+ .defaultValue(TableProperties.ORC_COMPRESSION_STRATEGY_DEFAULT)
+ .parse();
+ }
+
+ public DistributionMode distributionMode() {
+ String modeName =
+ confParser
+ .stringConf()
+ .option(FlinkWriteOptions.DISTRIBUTION_MODE.key())
+ .flinkConfig(FlinkWriteOptions.DISTRIBUTION_MODE)
+ .tableProperty(TableProperties.WRITE_DISTRIBUTION_MODE)
+ .defaultValue(TableProperties.WRITE_DISTRIBUTION_MODE_NONE)
+ .parse();
+ return DistributionMode.fromName(modeName);
+ }
+
+ public int workerPoolSize() {
+ return confParser
+ .intConf()
+ .flinkConfig(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE)
+ .defaultValue(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.defaultValue())
+ .parse();
+ }
+
+ public String branch() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.BRANCH.key())
+ .defaultValue(FlinkWriteOptions.BRANCH.defaultValue())
+ .parse();
+ }
+
+ public Integer writeParallelism() {
+ return confParser.intConf().option(FlinkWriteOptions.WRITE_PARALLELISM.key()).parseOptional();
+ }
+
+ /**
+ * NOTE: This may be removed or changed in a future release. This value specifies the interval for
+ * refreshing the table instances in sink writer subtasks. If not specified then the default
+ * behavior is to not refresh the table.
+ *
+ * @return the interval for refreshing the table in sink writer subtasks
+ */
+ @Experimental
+ public Duration tableRefreshInterval() {
+ return confParser
+ .durationConf()
+ .option(FlinkWriteOptions.TABLE_REFRESH_INTERVAL.key())
+ .flinkConfig(FlinkWriteOptions.TABLE_REFRESH_INTERVAL)
+ .parseOptional();
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java
new file mode 100644
index 0000000000..df73f2e09c
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.iceberg.SnapshotRef;
+
+/** Flink sink write options */
+public class FlinkWriteOptions {
+
+ private FlinkWriteOptions() {}
+
+ // File format for write operations(default: Table write.format.default )
+ public static final ConfigOption<String> WRITE_FORMAT =
+ ConfigOptions.key("write-format").stringType().noDefaultValue();
+
+ // Overrides this table's write.target-file-size-bytes
+ public static final ConfigOption<Long> TARGET_FILE_SIZE_BYTES =
+ ConfigOptions.key("target-file-size-bytes").longType().noDefaultValue();
+
+ // Overrides this table's write.<FILE_FORMAT>.compression-codec
+ public static final ConfigOption<String> COMPRESSION_CODEC =
+ ConfigOptions.key("compression-codec").stringType().noDefaultValue();
+
+ // Overrides this table's write.<FILE_FORMAT>.compression-level
+ public static final ConfigOption<String> COMPRESSION_LEVEL =
+ ConfigOptions.key("compression-level").stringType().noDefaultValue();
+
+ // Overrides this table's write.<FILE_FORMAT>.compression-strategy
+ public static final ConfigOption<String> COMPRESSION_STRATEGY =
+ ConfigOptions.key("compression-strategy").stringType().noDefaultValue();
+
+ // Overrides this table's write.upsert.enabled
+ public static final ConfigOption<Boolean> WRITE_UPSERT_ENABLED =
+ ConfigOptions.key("upsert-enabled").booleanType().noDefaultValue();
+
+ public static final ConfigOption<Boolean> OVERWRITE_MODE =
+ ConfigOptions.key("overwrite-enabled").booleanType().defaultValue(false);
+
+ // Overrides the table's write.distribution-mode
+ public static final ConfigOption<String> DISTRIBUTION_MODE =
+ ConfigOptions.key("distribution-mode").stringType().noDefaultValue();
+
+ // Branch to write to
+ public static final ConfigOption<String> BRANCH =
+ ConfigOptions.key("branch").stringType().defaultValue(SnapshotRef.MAIN_BRANCH);
+
+ public static final ConfigOption<Integer> WRITE_PARALLELISM =
+ ConfigOptions.key("write-parallelism").intType().noDefaultValue();
+
+ @Experimental
+ public static final ConfigOption<Duration> TABLE_REFRESH_INTERVAL =
+ ConfigOptions.key("table-refresh-interval").durationType().noDefaultValue();
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java
new file mode 100644
index 0000000000..1b9268569d
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.api.constraints.UniqueConstraint;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.ProviderContext;
+import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.abilities.SupportsOverwrite;
+import org.apache.flink.table.connector.sink.abilities.SupportsPartitioning;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Preconditions;
+import org.apache.iceberg.flink.sink.FlinkSink;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+
+public class IcebergTableSink implements DynamicTableSink, SupportsPartitioning, SupportsOverwrite {
+ private final TableLoader tableLoader;
+ private final TableSchema tableSchema;
+ private final ReadableConfig readableConfig;
+ private final Map<String, String> writeProps;
+
+ private boolean overwrite = false;
+
+ private IcebergTableSink(IcebergTableSink toCopy) {
+ this.tableLoader = toCopy.tableLoader;
+ this.tableSchema = toCopy.tableSchema;
+ this.overwrite = toCopy.overwrite;
+ this.readableConfig = toCopy.readableConfig;
+ this.writeProps = toCopy.writeProps;
+ }
+
+ public IcebergTableSink(
+ TableLoader tableLoader,
+ TableSchema tableSchema,
+ ReadableConfig readableConfig,
+ Map<String, String> writeProps) {
+ this.tableLoader = tableLoader;
+ this.tableSchema = tableSchema;
+ this.readableConfig = readableConfig;
+ this.writeProps = writeProps;
+ }
+
+ @Override
+ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
+ Preconditions.checkState(
+ !overwrite || context.isBounded(),
+ "Unbounded data stream doesn't support overwrite operation.");
+
+ List<String> equalityColumns =
+ tableSchema.getPrimaryKey().map(UniqueConstraint::getColumns).orElseGet(ImmutableList::of);
+
+ return new DataStreamSinkProvider() {
+ @Override
+ public DataStreamSink<?> consumeDataStream(
+ ProviderContext providerContext, DataStream<RowData> dataStream) {
+ return FlinkSink.forRowData(dataStream)
+ .tableLoader(tableLoader)
+ .tableSchema(tableSchema)
+ .equalityFieldColumns(equalityColumns)
+ .overwrite(overwrite)
+ .setAll(writeProps)
+ .flinkConf(readableConfig)
+ .append();
+ }
+ };
+ }
+
+ @Override
+ public void applyStaticPartition(Map<String, String> partition) {
+ // The flink's PartitionFanoutWriter will handle the static partition write policy
+ // automatically.
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode(ChangelogMode requestedMode) {
+ ChangelogMode.Builder builder = ChangelogMode.newBuilder();
+ for (RowKind kind : requestedMode.getContainedKinds()) {
+ builder.addContainedKind(kind);
+ }
+ return builder.build();
+ }
+
+ @Override
+ public DynamicTableSink copy() {
+ return new IcebergTableSink(this);
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "Iceberg table sink";
+ }
+
+ @Override
+ public void applyOverwrite(boolean newOverwrite) {
+ this.overwrite = newOverwrite;
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java
new file mode 100644
index 0000000000..d4cec7a3e8
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.lang.reflect.Array;
+import java.nio.ByteBuffer;
+import java.time.LocalDateTime;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
+import org.apache.iceberg.util.UUIDUtil;
+
+public class RowDataWrapper implements StructLike {
+
+ private final LogicalType[] types;
+ private final PositionalGetter<?>[] getters;
+ private RowData rowData = null;
+
+ public RowDataWrapper(RowType rowType, Types.StructType struct) {
+ int size = rowType.getFieldCount();
+
+ types = (LogicalType[]) Array.newInstance(LogicalType.class, size);
+ getters = (PositionalGetter[]) Array.newInstance(PositionalGetter.class, size);
+
+ for (int i = 0; i < size; i++) {
+ types[i] = rowType.getTypeAt(i);
+ getters[i] = buildGetter(types[i], struct.fields().get(i).type());
+ }
+ }
+
+ public RowDataWrapper wrap(RowData data) {
+ this.rowData = data;
+ return this;
+ }
+
+ @Override
+ public int size() {
+ return types.length;
+ }
+
+ @Override
+ public <T> T get(int pos, Class<T> javaClass) {
+ if (rowData.isNullAt(pos)) {
+ return null;
+ } else if (getters[pos] != null) {
+ return javaClass.cast(getters[pos].get(rowData, pos));
+ }
+
+ Object value = RowData.createFieldGetter(types[pos], pos).getFieldOrNull(rowData);
+ return javaClass.cast(value);
+ }
+
+ @Override
+ public <T> void set(int pos, T value) {
+ throw new UnsupportedOperationException(
+ "Could not set a field in the RowDataWrapper because rowData is read-only");
+ }
+
+ private interface PositionalGetter<T> {
+ T get(RowData data, int pos);
+ }
+
+ private static PositionalGetter<?> buildGetter(LogicalType logicalType, Type type) {
+ switch (logicalType.getTypeRoot()) {
+ case TINYINT:
+ return (row, pos) -> (int) row.getByte(pos);
+ case SMALLINT:
+ return (row, pos) -> (int) row.getShort(pos);
+ case CHAR:
+ case VARCHAR:
+ return (row, pos) -> row.getString(pos).toString();
+
+ case BINARY:
+ case VARBINARY:
+ if (Type.TypeID.UUID == type.typeId()) {
+ return (row, pos) -> UUIDUtil.convert(row.getBinary(pos));
+ } else {
+ return (row, pos) -> ByteBuffer.wrap(row.getBinary(pos));
+ }
+
+ case DECIMAL:
+ DecimalType decimalType = (DecimalType) logicalType;
+ return (row, pos) ->
+ row.getDecimal(pos, decimalType.getPrecision(), decimalType.getScale()).toBigDecimal();
+
+ case TIME_WITHOUT_TIME_ZONE:
+ // Time in RowData is in milliseconds (Integer), while iceberg's time is microseconds
+ // (Long).
+ return (row, pos) -> ((long) row.getInt(pos)) * 1_000;
+
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ TimestampType timestampType = (TimestampType) logicalType;
+ return (row, pos) -> {
+ LocalDateTime localDateTime =
+ row.getTimestamp(pos, timestampType.getPrecision()).toLocalDateTime();
+ return DateTimeUtil.microsFromTimestamp(localDateTime);
+ };
+
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ LocalZonedTimestampType lzTs = (LocalZonedTimestampType) logicalType;
+ return (row, pos) -> {
+ TimestampData timestampData = row.getTimestamp(pos, lzTs.getPrecision());
+ return timestampData.getMillisecond() * 1000
+ + timestampData.getNanoOfMillisecond() / 1000;
+ };
+
+ case ROW:
+ RowType rowType = (RowType) logicalType;
+ Types.StructType structType = (Types.StructType) type;
+
+ RowDataWrapper nestedWrapper = new RowDataWrapper(rowType, structType);
+ return (row, pos) -> nestedWrapper.wrap(row.getRow(pos, rowType.getFieldCount()));
+
+ default:
+ return null;
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java
new file mode 100644
index 0000000000..da509451fe
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.hadoop.SerializableConfiguration;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+
+/**
+ * Serializable loader to load an Iceberg {@link Table}. Flink needs to get {@link Table} objects in
+ * the cluster (for example, to get splits), not just on the client side. So we need an Iceberg
+ * table loader to get the {@link Table} object.
+ */
+public interface TableLoader extends Closeable, Serializable, Cloneable {
+
+ void open();
+
+ boolean isOpen();
+
+ Table loadTable();
+
+ /** Clone a TableLoader */
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ TableLoader clone();
+
+ static TableLoader fromCatalog(CatalogLoader catalogLoader, TableIdentifier identifier) {
+ return new CatalogTableLoader(catalogLoader, identifier);
+ }
+
+ static TableLoader fromHadoopTable(String location) {
+ return fromHadoopTable(location, FlinkCatalogFactory.clusterHadoopConf());
+ }
+
+ static TableLoader fromHadoopTable(String location, Configuration hadoopConf) {
+ return new HadoopTableLoader(location, hadoopConf);
+ }
+
+ class HadoopTableLoader implements TableLoader {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String location;
+ private final SerializableConfiguration hadoopConf;
+
+ private transient HadoopTables tables;
+
+ private HadoopTableLoader(String location, Configuration conf) {
+ this.location = location;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ }
+
+ @Override
+ public void open() {
+ tables = new HadoopTables(hadoopConf.get());
+ }
+
+ @Override
+ public boolean isOpen() {
+ return tables != null;
+ }
+
+ @Override
+ public Table loadTable() {
+ FlinkEnvironmentContext.init();
+ return tables.load(location);
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public TableLoader clone() {
+ return new HadoopTableLoader(location, new Configuration(hadoopConf.get()));
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("location", location).toString();
+ }
+ }
+
+ class CatalogTableLoader implements TableLoader {
+
+ private static final long serialVersionUID = 1L;
+
+ private final CatalogLoader catalogLoader;
+ private final String identifier;
+
+ private transient Catalog catalog;
+
+ private CatalogTableLoader(CatalogLoader catalogLoader, TableIdentifier tableIdentifier) {
+ this.catalogLoader = catalogLoader;
+ this.identifier = tableIdentifier.toString();
+ }
+
+ @Override
+ public void open() {
+ catalog = catalogLoader.loadCatalog();
+ }
+
+ @Override
+ public boolean isOpen() {
+ return catalog != null;
+ }
+
+ @Override
+ public Table loadTable() {
+ FlinkEnvironmentContext.init();
+ return catalog.loadTable(TableIdentifier.parse(identifier));
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (catalog instanceof Closeable) {
+ ((Closeable) catalog).close();
+ }
+
+ catalog = null;
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public TableLoader clone() {
+ return new CatalogTableLoader(catalogLoader.clone(), TableIdentifier.parse(identifier));
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("tableIdentifier", identifier)
+ .add("catalogLoader", catalogLoader)
+ .toString();
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java
new file mode 100644
index 0000000000..f8f1b74b1c
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.List;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.BigIntType;
+import org.apache.flink.table.types.logical.BinaryType;
+import org.apache.flink.table.types.logical.BooleanType;
+import org.apache.flink.table.types.logical.DateType;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.DoubleType;
+import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.TimeType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.table.types.logical.VarBinaryType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+class TypeToFlinkType extends TypeUtil.SchemaVisitor<LogicalType> {
+ TypeToFlinkType() {}
+
+ @Override
+ public LogicalType schema(Schema schema, LogicalType structType) {
+ return structType;
+ }
+
+ @Override
+ public LogicalType struct(Types.StructType struct, List<LogicalType> fieldResults) {
+ List<Types.NestedField> fields = struct.fields();
+
+ List<RowType.RowField> flinkFields = Lists.newArrayListWithExpectedSize(fieldResults.size());
+ for (int i = 0; i < fields.size(); i += 1) {
+ Types.NestedField field = fields.get(i);
+ LogicalType type = fieldResults.get(i);
+ RowType.RowField flinkField =
+ new RowType.RowField(field.name(), type.copy(field.isOptional()), field.doc());
+ flinkFields.add(flinkField);
+ }
+
+ return new RowType(flinkFields);
+ }
+
+ @Override
+ public LogicalType field(Types.NestedField field, LogicalType fieldResult) {
+ return fieldResult;
+ }
+
+ @Override
+ public LogicalType list(Types.ListType list, LogicalType elementResult) {
+ return new ArrayType(elementResult.copy(list.isElementOptional()));
+ }
+
+ @Override
+ public LogicalType map(Types.MapType map, LogicalType keyResult, LogicalType valueResult) {
+ // keys in map are not allowed to be null.
+ return new MapType(keyResult.copy(false), valueResult.copy(map.isValueOptional()));
+ }
+
+ @Override
+ public LogicalType primitive(Type.PrimitiveType primitive) {
+ switch (primitive.typeId()) {
+ case BOOLEAN:
+ return new BooleanType();
+ case INTEGER:
+ return new IntType();
+ case LONG:
+ return new BigIntType();
+ case FLOAT:
+ return new FloatType();
+ case DOUBLE:
+ return new DoubleType();
+ case DATE:
+ return new DateType();
+ case TIME:
+ // For the type: Flink only support TimeType with default precision (second) now. The
+ // precision of time is
+ // not supported in Flink, so we can think of it as a simple time type directly.
+ // For the data: Flink uses int that support mills to represent time data, so it supports
+ // mills precision.
+ return new TimeType();
+ case TIMESTAMP:
+ Types.TimestampType timestamp = (Types.TimestampType) primitive;
+ if (timestamp.shouldAdjustToUTC()) {
+ // MICROS
+ return new LocalZonedTimestampType(6);
+ } else {
+ // MICROS
+ return new TimestampType(6);
+ }
+ case STRING:
+ return new VarCharType(VarCharType.MAX_LENGTH);
+ case UUID:
+ // UUID length is 16
+ return new BinaryType(16);
+ case FIXED:
+ Types.FixedType fixedType = (Types.FixedType) primitive;
+ return new BinaryType(fixedType.length());
+ case BINARY:
+ return new VarBinaryType(VarBinaryType.MAX_LENGTH);
+ case DECIMAL:
+ Types.DecimalType decimal = (Types.DecimalType) primitive;
+ return new DecimalType(decimal.precision(), decimal.scale());
+ default:
+ throw new UnsupportedOperationException(
+ "Cannot convert unknown type to Flink: " + primitive);
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java
new file mode 100644
index 0000000000..06ac54617a
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.actions;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.iceberg.Table;
+
+public class Actions {
+
+ public static final Configuration CONFIG =
+ new Configuration()
+ // disable classloader check as Avro may cache class/object in the serializers.
+ .set(CoreOptions.CHECK_LEAKED_CLASSLOADER, false);
+
+ private StreamExecutionEnvironment env;
+ private Table table;
+
+ private Actions(StreamExecutionEnvironment env, Table table) {
+ this.env = env;
+ this.table = table;
+ }
+
+ public static Actions forTable(StreamExecutionEnvironment env, Table table) {
+ return new Actions(env, table);
+ }
+
+ public static Actions forTable(Table table) {
+ return new Actions(StreamExecutionEnvironment.getExecutionEnvironment(CONFIG), table);
+ }
+
+ public RewriteDataFilesAction rewriteDataFiles() {
+ return new RewriteDataFilesAction(env, table);
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java
new file mode 100644
index 0000000000..9876bb3861
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.actions;
+
+import java.util.List;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.iceberg.CombinedScanTask;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.BaseRewriteDataFilesAction;
+import org.apache.iceberg.flink.source.RowDataRewriter;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+public class RewriteDataFilesAction extends BaseRewriteDataFilesAction<RewriteDataFilesAction> {
+
+ private StreamExecutionEnvironment env;
+ private int maxParallelism;
+
+ public RewriteDataFilesAction(StreamExecutionEnvironment env, Table table) {
+ super(table);
+ this.env = env;
+ this.maxParallelism = env.getParallelism();
+ }
+
+ @Override
+ protected FileIO fileIO() {
+ return table().io();
+ }
+
+ @Override
+ protected List<DataFile> rewriteDataForTasks(List<CombinedScanTask> combinedScanTasks) {
+ int size = combinedScanTasks.size();
+ int parallelism = Math.min(size, maxParallelism);
+ DataStream<CombinedScanTask> dataStream = env.fromCollection(combinedScanTasks);
+ RowDataRewriter rowDataRewriter =
+ new RowDataRewriter(table(), caseSensitive(), fileIO(), encryptionManager());
+ try {
+ return rowDataRewriter.rewriteDataForTasks(dataStream, parallelism);
+ } catch (Exception e) {
+ throw new RuntimeException("Rewrite data file error.", e);
+ }
+ }
+
+ @Override
+ protected RewriteDataFilesAction self() {
+ return this;
+ }
+
+ public RewriteDataFilesAction maxParallelism(int parallelism) {
+ Preconditions.checkArgument(parallelism > 0, "Invalid max parallelism %s", parallelism);
+ this.maxParallelism = parallelism;
+ return this;
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java
new file mode 100644
index 0000000000..8103224a0b
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.NullType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.avro.AvroWithPartnerByStructureVisitor;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.util.Pair;
+
+public abstract class AvroWithFlinkSchemaVisitor<T>
+ extends AvroWithPartnerByStructureVisitor<LogicalType, T> {
+
+ @Override
+ protected boolean isStringType(LogicalType logicalType) {
+ return logicalType.getTypeRoot().getFamilies().contains(LogicalTypeFamily.CHARACTER_STRING);
+ }
+
+ @Override
+ protected boolean isMapType(LogicalType logicalType) {
+ return logicalType instanceof MapType;
+ }
+
+ @Override
+ protected LogicalType arrayElementType(LogicalType arrayType) {
+ Preconditions.checkArgument(
+ arrayType instanceof ArrayType, "Invalid array: %s is not an array", arrayType);
+ return ((ArrayType) arrayType).getElementType();
+ }
+
+ @Override
+ protected LogicalType mapKeyType(LogicalType mapType) {
+ Preconditions.checkArgument(isMapType(mapType), "Invalid map: %s is not a map", mapType);
+ return ((MapType) mapType).getKeyType();
+ }
+
+ @Override
+ protected LogicalType mapValueType(LogicalType mapType) {
+ Preconditions.checkArgument(isMapType(mapType), "Invalid map: %s is not a map", mapType);
+ return ((MapType) mapType).getValueType();
+ }
+
+ @Override
+ protected Pair<String, LogicalType> fieldNameAndType(LogicalType structType, int pos) {
+ Preconditions.checkArgument(
+ structType instanceof RowType, "Invalid struct: %s is not a struct", structType);
+ RowType.RowField field = ((RowType) structType).getFields().get(pos);
+ return Pair.of(field.getName(), field.getType());
+ }
+
+ @Override
+ protected LogicalType nullType() {
+ return new NullType();
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroReader.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroReader.java
new file mode 100644
index 0000000000..8640495973
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroReader.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+import org.apache.avro.LogicalType;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.Decoder;
+import org.apache.flink.table.data.RowData;
+import org.apache.iceberg.avro.AvroSchemaWithTypeVisitor;
+import org.apache.iceberg.avro.SupportsRowPosition;
+import org.apache.iceberg.avro.ValueReader;
+import org.apache.iceberg.avro.ValueReaders;
+import org.apache.iceberg.data.avro.DecoderResolver;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+public class FlinkAvroReader implements DatumReader<RowData>, SupportsRowPosition {
+
+ private final Schema readSchema;
+ private final ValueReader<RowData> reader;
+ private Schema fileSchema = null;
+
+ public FlinkAvroReader(org.apache.iceberg.Schema expectedSchema, Schema readSchema) {
+ this(expectedSchema, readSchema, ImmutableMap.of());
+ }
+
+ @SuppressWarnings("unchecked")
+ public FlinkAvroReader(
+ org.apache.iceberg.Schema expectedSchema, Schema readSchema, Map<Integer, ?> constants) {
+ this.readSchema = readSchema;
+ this.reader =
+ (ValueReader<RowData>)
+ AvroSchemaWithTypeVisitor.visit(expectedSchema, readSchema, new ReadBuilder(constants));
+ }
+
+ @Override
+ public void setSchema(Schema newFileSchema) {
+ this.fileSchema = Schema.applyAliases(newFileSchema, readSchema);
+ }
+
+ @Override
+ public RowData read(RowData reuse, Decoder decoder) throws IOException {
+ return DecoderResolver.resolveAndRead(decoder, readSchema, fileSchema, reader, reuse);
+ }
+
+ @Override
+ public void setRowPositionSupplier(Supplier<Long> posSupplier) {
+ if (reader instanceof SupportsRowPosition) {
+ ((SupportsRowPosition) reader).setRowPositionSupplier(posSupplier);
+ }
+ }
+
+ private static class ReadBuilder extends AvroSchemaWithTypeVisitor<ValueReader<?>> {
+ private final Map<Integer, ?> idToConstant;
+
+ private ReadBuilder(Map<Integer, ?> idToConstant) {
+ this.idToConstant = idToConstant;
+ }
+
+ @Override
+ public ValueReader<?> record(
+ Types.StructType expected, Schema record, List<String> names, List<ValueReader<?>> fields) {
+ return FlinkValueReaders.struct(fields, expected.asStructType(), idToConstant);
+ }
+
+ @Override
+ public ValueReader<?> union(Type expected, Schema union, List<ValueReader<?>> options) {
+ return ValueReaders.union(options);
+ }
+
+ @Override
+ public ValueReader<?> array(
+ Types.ListType expected, Schema array, ValueReader<?> elementReader) {
+ return FlinkValueReaders.array(elementReader);
+ }
+
+ @Override
+ public ValueReader<?> map(
+ Types.MapType expected, Schema map, ValueReader<?> keyReader, ValueReader<?> valueReader) {
+ return FlinkValueReaders.arrayMap(keyReader, valueReader);
+ }
+
+ @Override
+ public ValueReader<?> map(Types.MapType expected, Schema map, ValueReader<?> valueReader) {
+ return FlinkValueReaders.map(FlinkValueReaders.strings(), valueReader);
+ }
+
+ @Override
+ public ValueReader<?> primitive(Type.PrimitiveType expected, Schema primitive) {
+ LogicalType logicalType = primitive.getLogicalType();
+ if (logicalType != null) {
+ switch (logicalType.getName()) {
+ case "date":
+ return ValueReaders.ints();
+
+ case "time-micros":
+ return FlinkValueReaders.timeMicros();
+
+ case "timestamp-millis":
+ return FlinkValueReaders.timestampMills();
+
+ case "timestamp-micros":
+ return FlinkValueReaders.timestampMicros();
+
+ case "decimal":
+ LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
+ return FlinkValueReaders.decimal(
+ ValueReaders.decimalBytesReader(primitive),
+ decimal.getPrecision(),
+ decimal.getScale());
+
+ case "uuid":
+ return FlinkValueReaders.uuids();
+
+ default:
+ throw new IllegalArgumentException("Unknown logical type: " + logicalType);
+ }
+ }
+
+ switch (primitive.getType()) {
+ case NULL:
+ return ValueReaders.nulls();
+ case BOOLEAN:
+ return ValueReaders.booleans();
+ case INT:
+ return ValueReaders.ints();
+ case LONG:
+ return ValueReaders.longs();
+ case FLOAT:
+ return ValueReaders.floats();
+ case DOUBLE:
+ return ValueReaders.doubles();
+ case STRING:
+ return FlinkValueReaders.strings();
+ case FIXED:
+ return ValueReaders.fixed(primitive.getFixedSize());
+ case BYTES:
+ return ValueReaders.bytes();
+ case ENUM:
+ return FlinkValueReaders.enums(primitive.getEnumSymbols());
+ default:
+ throw new IllegalArgumentException("Unsupported type: " + primitive);
+ }
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java
new file mode 100644
index 0000000000..873e657831
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.apache.avro.io.Encoder;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.FieldMetrics;
+import org.apache.iceberg.avro.MetricsAwareDatumWriter;
+import org.apache.iceberg.avro.ValueWriter;
+import org.apache.iceberg.avro.ValueWriters;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+public class FlinkAvroWriter implements MetricsAwareDatumWriter<RowData> {
+ private final RowType rowType;
+ private ValueWriter<RowData> writer = null;
+
+ public FlinkAvroWriter(RowType rowType) {
+ this.rowType = rowType;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void setSchema(Schema schema) {
+ this.writer =
+ (ValueWriter<RowData>)
+ AvroWithFlinkSchemaVisitor.visit(rowType, schema, new WriteBuilder());
+ }
+
+ @Override
+ public void write(RowData datum, Encoder out) throws IOException {
+ writer.write(datum, out);
+ }
+
+ @Override
+ public Stream<FieldMetrics> metrics() {
+ return writer.metrics();
+ }
+
+ private static class WriteBuilder extends AvroWithFlinkSchemaVisitor<ValueWriter<?>> {
+ @Override
+ public ValueWriter<?> record(
+ LogicalType struct, Schema record, List<String> names, List<ValueWriter<?>> fields) {
+ return FlinkValueWriters.row(
+ fields,
+ IntStream.range(0, names.size())
+ .mapToObj(i -> fieldNameAndType(struct, i).second())
+ .collect(Collectors.toList()));
+ }
+
+ @Override
+ public ValueWriter<?> union(LogicalType type, Schema union, List<ValueWriter<?>> options) {
+ Preconditions.checkArgument(
+ options.contains(ValueWriters.nulls()),
+ "Cannot create writer for non-option union: %s",
+ union);
+ Preconditions.checkArgument(
+ options.size() == 2, "Cannot create writer for non-option union: %s", union);
+ if (union.getTypes().get(0).getType() == Schema.Type.NULL) {
+ return ValueWriters.option(0, options.get(1));
+ } else {
+ return ValueWriters.option(1, options.get(0));
+ }
+ }
+
+ @Override
+ public ValueWriter<?> array(LogicalType sArray, Schema array, ValueWriter<?> elementWriter) {
+ return FlinkValueWriters.array(elementWriter, arrayElementType(sArray));
+ }
+
+ @Override
+ public ValueWriter<?> map(LogicalType sMap, Schema map, ValueWriter<?> valueReader) {
+ return FlinkValueWriters.map(
+ FlinkValueWriters.strings(), mapKeyType(sMap), valueReader, mapValueType(sMap));
+ }
+
+ @Override
+ public ValueWriter<?> map(
+ LogicalType sMap, Schema map, ValueWriter<?> keyWriter, ValueWriter<?> valueWriter) {
+ return FlinkValueWriters.arrayMap(
+ keyWriter, mapKeyType(sMap), valueWriter, mapValueType(sMap));
+ }
+
+ @Override
+ public ValueWriter<?> primitive(LogicalType type, Schema primitive) {
+ org.apache.avro.LogicalType logicalType = primitive.getLogicalType();
+ if (logicalType != null) {
+ switch (logicalType.getName()) {
+ case "date":
+ return ValueWriters.ints();
+
+ case "time-micros":
+ return FlinkValueWriters.timeMicros();
+
+ case "timestamp-micros":
+ return FlinkValueWriters.timestampMicros();
+
+ case "decimal":
+ LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
+ return FlinkValueWriters.decimal(decimal.getPrecision(), decimal.getScale());
+
+ case "uuid":
+ return ValueWriters.uuids();
+
+ default:
+ throw new IllegalArgumentException("Unsupported logical type: " + logicalType);
+ }
+ }
+
+ switch (primitive.getType()) {
+ case NULL:
+ return ValueWriters.nulls();
+ case BOOLEAN:
+ return ValueWriters.booleans();
+ case INT:
+ switch (type.getTypeRoot()) {
+ case TINYINT:
+ return ValueWriters.tinyints();
+ case SMALLINT:
+ return ValueWriters.shorts();
+ default:
+ return ValueWriters.ints();
+ }
+ case LONG:
+ return ValueWriters.longs();
+ case FLOAT:
+ return ValueWriters.floats();
+ case DOUBLE:
+ return ValueWriters.doubles();
+ case STRING:
+ return FlinkValueWriters.strings();
+ case FIXED:
+ return ValueWriters.fixed(primitive.getFixedSize());
+ case BYTES:
+ return ValueWriters.bytes();
+ default:
+ throw new IllegalArgumentException("Unsupported type: " + primitive);
+ }
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java
new file mode 100644
index 0000000000..65b9d44ad4
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.orc.OrcRowReader;
+import org.apache.iceberg.orc.OrcSchemaWithTypeVisitor;
+import org.apache.iceberg.orc.OrcValueReader;
+import org.apache.iceberg.orc.OrcValueReaders;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.storage.ql.exec.vector.StructColumnVector;
+import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
+
+public class FlinkOrcReader implements OrcRowReader<RowData> {
+ private final OrcValueReader<?> reader;
+
+ public FlinkOrcReader(Schema iSchema, TypeDescription readSchema) {
+ this(iSchema, readSchema, ImmutableMap.of());
+ }
+
+ public FlinkOrcReader(Schema iSchema, TypeDescription readSchema, Map<Integer, ?> idToConstant) {
+ this.reader =
+ OrcSchemaWithTypeVisitor.visit(iSchema, readSchema, new ReadBuilder(idToConstant));
+ }
+
+ @Override
+ public RowData read(VectorizedRowBatch batch, int row) {
+ return (RowData) reader.read(new StructColumnVector(batch.size, batch.cols), row);
+ }
+
+ @Override
+ public void setBatchContext(long batchOffsetInFile) {
+ reader.setBatchContext(batchOffsetInFile);
+ }
+
+ private static class ReadBuilder extends OrcSchemaWithTypeVisitor<OrcValueReader<?>> {
+ private final Map<Integer, ?> idToConstant;
+
+ private ReadBuilder(Map<Integer, ?> idToConstant) {
+ this.idToConstant = idToConstant;
+ }
+
+ @Override
+ public OrcValueReader<RowData> record(
+ Types.StructType iStruct,
+ TypeDescription record,
+ List<String> names,
+ List<OrcValueReader<?>> fields) {
+ return FlinkOrcReaders.struct(fields, iStruct, idToConstant);
+ }
+
+ @Override
+ public OrcValueReader<ArrayData> list(
+ Types.ListType iList, TypeDescription array, OrcValueReader<?> elementReader) {
+ return FlinkOrcReaders.array(elementReader);
+ }
+
+ @Override
+ public OrcValueReader<MapData> map(
+ Types.MapType iMap,
+ TypeDescription map,
+ OrcValueReader<?> keyReader,
+ OrcValueReader<?> valueReader) {
+ return FlinkOrcReaders.map(keyReader, valueReader);
+ }
+
+ @Override
+ public OrcValueReader<?> primitive(Type.PrimitiveType iPrimitive, TypeDescription primitive) {
+ switch (iPrimitive.typeId()) {
+ case BOOLEAN:
+ return OrcValueReaders.booleans();
+ case INTEGER:
+ return OrcValueReaders.ints();
+ case LONG:
+ return OrcValueReaders.longs();
+ case FLOAT:
+ return OrcValueReaders.floats();
+ case DOUBLE:
+ return OrcValueReaders.doubles();
+ case DATE:
+ return FlinkOrcReaders.dates();
+ case TIME:
+ return FlinkOrcReaders.times();
+ case TIMESTAMP:
+ Types.TimestampType timestampType = (Types.TimestampType) iPrimitive;
+ if (timestampType.shouldAdjustToUTC()) {
+ return FlinkOrcReaders.timestampTzs();
+ } else {
+ return FlinkOrcReaders.timestamps();
+ }
+ case STRING:
+ return FlinkOrcReaders.strings();
+ case UUID:
+ case FIXED:
+ case BINARY:
+ return OrcValueReaders.bytes();
+ case DECIMAL:
+ Types.DecimalType decimalType = (Types.DecimalType) iPrimitive;
+ return FlinkOrcReaders.decimals(decimalType.precision(), decimalType.scale());
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid iceberg type %s corresponding to ORC type %s", iPrimitive, primitive));
+ }
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java
new file mode 100644
index 0000000000..7a4a15c7e6
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericArrayData;
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.iceberg.orc.OrcValueReader;
+import org.apache.iceberg.orc.OrcValueReaders;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.orc.storage.ql.exec.vector.BytesColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ColumnVector;
+import org.apache.orc.storage.ql.exec.vector.DecimalColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ListColumnVector;
+import org.apache.orc.storage.ql.exec.vector.LongColumnVector;
+import org.apache.orc.storage.ql.exec.vector.MapColumnVector;
+import org.apache.orc.storage.ql.exec.vector.TimestampColumnVector;
+import org.apache.orc.storage.serde2.io.HiveDecimalWritable;
+
+class FlinkOrcReaders {
+ private FlinkOrcReaders() {}
+
+ static OrcValueReader<StringData> strings() {
+ return StringReader.INSTANCE;
+ }
+
+ static OrcValueReader<Integer> dates() {
+ return DateReader.INSTANCE;
+ }
+
+ static OrcValueReader<DecimalData> decimals(int precision, int scale) {
+ if (precision <= 18) {
+ return new Decimal18Reader(precision, scale);
+ } else if (precision <= 38) {
+ return new Decimal38Reader(precision, scale);
+ } else {
+ throw new IllegalArgumentException("Invalid precision: " + precision);
+ }
+ }
+
+ static OrcValueReader<Integer> times() {
+ return TimeReader.INSTANCE;
+ }
+
+ static OrcValueReader<TimestampData> timestamps() {
+ return TimestampReader.INSTANCE;
+ }
+
+ static OrcValueReader<TimestampData> timestampTzs() {
+ return TimestampTzReader.INSTANCE;
+ }
+
+ static <T> OrcValueReader<ArrayData> array(OrcValueReader<T> elementReader) {
+ return new ArrayReader<>(elementReader);
+ }
+
+ public static <K, V> OrcValueReader<MapData> map(
+ OrcValueReader<K> keyReader, OrcValueReader<V> valueReader) {
+ return new MapReader<>(keyReader, valueReader);
+ }
+
+ public static OrcValueReader<RowData> struct(
+ List<OrcValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
+ return new StructReader(readers, struct, idToConstant);
+ }
+
+ private static class StringReader implements OrcValueReader<StringData> {
+ private static final StringReader INSTANCE = new StringReader();
+
+ @Override
+ public StringData nonNullRead(ColumnVector vector, int row) {
+ BytesColumnVector bytesVector = (BytesColumnVector) vector;
+ return StringData.fromBytes(
+ bytesVector.vector[row], bytesVector.start[row], bytesVector.length[row]);
+ }
+ }
+
+ private static class DateReader implements OrcValueReader<Integer> {
+ private static final DateReader INSTANCE = new DateReader();
+
+ @Override
+ public Integer nonNullRead(ColumnVector vector, int row) {
+ return (int) ((LongColumnVector) vector).vector[row];
+ }
+ }
+
+ private static class Decimal18Reader implements OrcValueReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal18Reader(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData nonNullRead(ColumnVector vector, int row) {
+ HiveDecimalWritable value = ((DecimalColumnVector) vector).vector[row];
+
+ // The hive ORC writer may will adjust the scale of decimal data.
+ Preconditions.checkArgument(
+ value.precision() <= precision,
+ "Cannot read value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ value);
+
+ return DecimalData.fromUnscaledLong(value.serialize64(scale), precision, scale);
+ }
+ }
+
+ private static class Decimal38Reader implements OrcValueReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal38Reader(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData nonNullRead(ColumnVector vector, int row) {
+ BigDecimal value =
+ ((DecimalColumnVector) vector).vector[row].getHiveDecimal().bigDecimalValue();
+
+ Preconditions.checkArgument(
+ value.precision() <= precision,
+ "Cannot read value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ value);
+
+ return DecimalData.fromBigDecimal(value, precision, scale);
+ }
+ }
+
+ private static class TimeReader implements OrcValueReader<Integer> {
+ private static final TimeReader INSTANCE = new TimeReader();
+
+ @Override
+ public Integer nonNullRead(ColumnVector vector, int row) {
+ long micros = ((LongColumnVector) vector).vector[row];
+ // Flink only support time mills, just erase micros.
+ return (int) (micros / 1000);
+ }
+ }
+
+ private static class TimestampReader implements OrcValueReader<TimestampData> {
+ private static final TimestampReader INSTANCE = new TimestampReader();
+
+ @Override
+ public TimestampData nonNullRead(ColumnVector vector, int row) {
+ TimestampColumnVector tcv = (TimestampColumnVector) vector;
+ LocalDateTime localDate =
+ Instant.ofEpochSecond(Math.floorDiv(tcv.time[row], 1_000), tcv.nanos[row])
+ .atOffset(ZoneOffset.UTC)
+ .toLocalDateTime();
+ return TimestampData.fromLocalDateTime(localDate);
+ }
+ }
+
+ private static class TimestampTzReader implements OrcValueReader<TimestampData> {
+ private static final TimestampTzReader INSTANCE = new TimestampTzReader();
+
+ @Override
+ public TimestampData nonNullRead(ColumnVector vector, int row) {
+ TimestampColumnVector tcv = (TimestampColumnVector) vector;
+ Instant instant =
+ Instant.ofEpochSecond(Math.floorDiv(tcv.time[row], 1_000), tcv.nanos[row])
+ .atOffset(ZoneOffset.UTC)
+ .toInstant();
+ return TimestampData.fromInstant(instant);
+ }
+ }
+
+ private static class ArrayReader<T> implements OrcValueReader<ArrayData> {
+ private final OrcValueReader<T> elementReader;
+
+ private ArrayReader(OrcValueReader<T> elementReader) {
+ this.elementReader = elementReader;
+ }
+
+ @Override
+ public ArrayData nonNullRead(ColumnVector vector, int row) {
+ ListColumnVector listVector = (ListColumnVector) vector;
+ int offset = (int) listVector.offsets[row];
+ int length = (int) listVector.lengths[row];
+ List<T> elements = Lists.newArrayListWithExpectedSize(length);
+ for (int c = 0; c < length; ++c) {
+ elements.add(elementReader.read(listVector.child, offset + c));
+ }
+ return new GenericArrayData(elements.toArray());
+ }
+
+ @Override
+ public void setBatchContext(long batchOffsetInFile) {
+ elementReader.setBatchContext(batchOffsetInFile);
+ }
+ }
+
+ private static class MapReader<K, V> implements OrcValueReader<MapData> {
+ private final OrcValueReader<K> keyReader;
+ private final OrcValueReader<V> valueReader;
+
+ private MapReader(OrcValueReader<K> keyReader, OrcValueReader<V> valueReader) {
+ this.keyReader = keyReader;
+ this.valueReader = valueReader;
+ }
+
+ @Override
+ public MapData nonNullRead(ColumnVector vector, int row) {
+ MapColumnVector mapVector = (MapColumnVector) vector;
+ int offset = (int) mapVector.offsets[row];
+ long length = mapVector.lengths[row];
+
+ Map<K, V> map = Maps.newHashMap();
+ for (int c = 0; c < length; c++) {
+ K key = keyReader.read(mapVector.keys, offset + c);
+ V value = valueReader.read(mapVector.values, offset + c);
+ map.put(key, value);
+ }
+
+ return new GenericMapData(map);
+ }
+
+ @Override
+ public void setBatchContext(long batchOffsetInFile) {
+ keyReader.setBatchContext(batchOffsetInFile);
+ valueReader.setBatchContext(batchOffsetInFile);
+ }
+ }
+
+ private static class StructReader extends OrcValueReaders.StructReader<RowData> {
+ private final int numFields;
+
+ StructReader(
+ List<OrcValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
+ super(readers, struct, idToConstant);
+ this.numFields = struct.fields().size();
+ }
+
+ @Override
+ protected RowData create() {
+ return new GenericRowData(numFields);
+ }
+
+ @Override
+ protected void set(RowData struct, int pos, Object value) {
+ ((GenericRowData) struct).setField(pos, value);
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java
new file mode 100644
index 0000000000..6a31accffd
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.Deque;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.FieldMetrics;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.data.orc.GenericOrcWriters;
+import org.apache.iceberg.orc.OrcRowWriter;
+import org.apache.iceberg.orc.OrcValueWriter;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
+
+public class FlinkOrcWriter implements OrcRowWriter<RowData> {
+ private final FlinkOrcWriters.RowDataWriter writer;
+
+ private FlinkOrcWriter(RowType rowType, Schema iSchema) {
+ this.writer =
+ (FlinkOrcWriters.RowDataWriter)
+ FlinkSchemaVisitor.visit(rowType, iSchema, new WriteBuilder());
+ }
+
+ public static OrcRowWriter<RowData> buildWriter(RowType rowType, Schema iSchema) {
+ return new FlinkOrcWriter(rowType, iSchema);
+ }
+
+ @Override
+ public void write(RowData row, VectorizedRowBatch output) {
+ Preconditions.checkArgument(row != null, "value must not be null");
+ writer.writeRow(row, output);
+ }
+
+ @Override
+ public List<OrcValueWriter<?>> writers() {
+ return writer.writers();
+ }
+
+ @Override
+ public Stream<FieldMetrics<?>> metrics() {
+ return writer.metrics();
+ }
+
+ private static class WriteBuilder extends FlinkSchemaVisitor<OrcValueWriter<?>> {
+ private final Deque<Integer> fieldIds = Lists.newLinkedList();
+
+ private WriteBuilder() {}
+
+ @Override
+ public void beforeField(Types.NestedField field) {
+ fieldIds.push(field.fieldId());
+ }
+
+ @Override
+ public void afterField(Types.NestedField field) {
+ fieldIds.pop();
+ }
+
+ @Override
+ public OrcValueWriter<RowData> record(
+ Types.StructType iStruct, List<OrcValueWriter<?>> results, List<LogicalType> fieldType) {
+ return FlinkOrcWriters.struct(results, fieldType);
+ }
+
+ @Override
+ public OrcValueWriter<?> map(
+ Types.MapType iMap,
+ OrcValueWriter<?> key,
+ OrcValueWriter<?> value,
+ LogicalType keyType,
+ LogicalType valueType) {
+ return FlinkOrcWriters.map(key, value, keyType, valueType);
+ }
+
+ @Override
+ public OrcValueWriter<?> list(
+ Types.ListType iList, OrcValueWriter<?> element, LogicalType elementType) {
+ return FlinkOrcWriters.list(element, elementType);
+ }
+
+ @Override
+ public OrcValueWriter<?> primitive(Type.PrimitiveType iPrimitive, LogicalType flinkPrimitive) {
+ switch (iPrimitive.typeId()) {
+ case BOOLEAN:
+ return GenericOrcWriters.booleans();
+ case INTEGER:
+ switch (flinkPrimitive.getTypeRoot()) {
+ case TINYINT:
+ return GenericOrcWriters.bytes();
+ case SMALLINT:
+ return GenericOrcWriters.shorts();
+ }
+ return GenericOrcWriters.ints();
+ case LONG:
+ return GenericOrcWriters.longs();
+ case FLOAT:
+ Preconditions.checkArgument(
+ fieldIds.peek() != null,
+ String.format(
+ "[BUG] Cannot find field id for primitive field with type %s. This is likely because id "
+ + "information is not properly pushed during schema visiting.",
+ iPrimitive));
+ return GenericOrcWriters.floats(fieldIds.peek());
+ case DOUBLE:
+ Preconditions.checkArgument(
+ fieldIds.peek() != null,
+ String.format(
+ "[BUG] Cannot find field id for primitive field with type %s. This is likely because id "
+ + "information is not properly pushed during schema visiting.",
+ iPrimitive));
+ return GenericOrcWriters.doubles(fieldIds.peek());
+ case DATE:
+ return FlinkOrcWriters.dates();
+ case TIME:
+ return FlinkOrcWriters.times();
+ case TIMESTAMP:
+ Types.TimestampType timestampType = (Types.TimestampType) iPrimitive;
+ if (timestampType.shouldAdjustToUTC()) {
+ return FlinkOrcWriters.timestampTzs();
+ } else {
+ return FlinkOrcWriters.timestamps();
+ }
+ case STRING:
+ return FlinkOrcWriters.strings();
+ case UUID:
+ case FIXED:
+ case BINARY:
+ return GenericOrcWriters.byteArrays();
+ case DECIMAL:
+ Types.DecimalType decimalType = (Types.DecimalType) iPrimitive;
+ return FlinkOrcWriters.decimals(decimalType.precision(), decimalType.scale());
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid iceberg type %s corresponding to Flink logical type %s",
+ iPrimitive, flinkPrimitive));
+ }
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java
new file mode 100644
index 0000000000..da2f95cf82
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.iceberg.FieldMetrics;
+import org.apache.iceberg.data.orc.GenericOrcWriters;
+import org.apache.iceberg.orc.OrcValueWriter;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.orc.storage.common.type.HiveDecimal;
+import org.apache.orc.storage.ql.exec.vector.BytesColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ColumnVector;
+import org.apache.orc.storage.ql.exec.vector.DecimalColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ListColumnVector;
+import org.apache.orc.storage.ql.exec.vector.LongColumnVector;
+import org.apache.orc.storage.ql.exec.vector.MapColumnVector;
+import org.apache.orc.storage.ql.exec.vector.TimestampColumnVector;
+
+class FlinkOrcWriters {
+
+ private FlinkOrcWriters() {}
+
+ static OrcValueWriter<StringData> strings() {
+ return StringWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<Integer> dates() {
+ return DateWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<Integer> times() {
+ return TimeWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<TimestampData> timestamps() {
+ return TimestampWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<TimestampData> timestampTzs() {
+ return TimestampTzWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<DecimalData> decimals(int precision, int scale) {
+ if (precision <= 18) {
+ return new Decimal18Writer(precision, scale);
+ } else if (precision <= 38) {
+ return new Decimal38Writer(precision, scale);
+ } else {
+ throw new IllegalArgumentException("Invalid precision: " + precision);
+ }
+ }
+
+ static <T> OrcValueWriter<ArrayData> list(
+ OrcValueWriter<T> elementWriter, LogicalType elementType) {
+ return new ListWriter<>(elementWriter, elementType);
+ }
+
+ static <K, V> OrcValueWriter<MapData> map(
+ OrcValueWriter<K> keyWriter,
+ OrcValueWriter<V> valueWriter,
+ LogicalType keyType,
+ LogicalType valueType) {
+ return new MapWriter<>(keyWriter, valueWriter, keyType, valueType);
+ }
+
+ static OrcValueWriter<RowData> struct(List<OrcValueWriter<?>> writers, List<LogicalType> types) {
+ return new RowDataWriter(writers, types);
+ }
+
+ private static class StringWriter implements OrcValueWriter<StringData> {
+ private static final StringWriter INSTANCE = new StringWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, StringData data, ColumnVector output) {
+ byte[] value = data.toBytes();
+ ((BytesColumnVector) output).setRef(rowId, value, 0, value.length);
+ }
+ }
+
+ private static class DateWriter implements OrcValueWriter<Integer> {
+ private static final DateWriter INSTANCE = new DateWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, Integer data, ColumnVector output) {
+ ((LongColumnVector) output).vector[rowId] = data;
+ }
+ }
+
+ private static class TimeWriter implements OrcValueWriter<Integer> {
+ private static final TimeWriter INSTANCE = new TimeWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, Integer millis, ColumnVector output) {
+ // The time in flink is in millisecond, while the standard time in iceberg is microsecond.
+ // So we need to transform it to microsecond.
+ ((LongColumnVector) output).vector[rowId] = millis * 1000L;
+ }
+ }
+
+ private static class TimestampWriter implements OrcValueWriter<TimestampData> {
+ private static final TimestampWriter INSTANCE = new TimestampWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
+ TimestampColumnVector cv = (TimestampColumnVector) output;
+ cv.setIsUTC(true);
+ // millis
+ OffsetDateTime offsetDateTime = data.toInstant().atOffset(ZoneOffset.UTC);
+ cv.time[rowId] =
+ offsetDateTime.toEpochSecond() * 1_000 + offsetDateTime.getNano() / 1_000_000;
+ // truncate nanos to only keep microsecond precision.
+ cv.nanos[rowId] = (offsetDateTime.getNano() / 1_000) * 1_000;
+ }
+ }
+
+ private static class TimestampTzWriter implements OrcValueWriter<TimestampData> {
+ private static final TimestampTzWriter INSTANCE = new TimestampTzWriter();
+
+ @SuppressWarnings("JavaInstantGetSecondsGetNano")
+ @Override
+ public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
+ TimestampColumnVector cv = (TimestampColumnVector) output;
+ // millis
+ Instant instant = data.toInstant();
+ cv.time[rowId] = instant.toEpochMilli();
+ // truncate nanos to only keep microsecond precision.
+ cv.nanos[rowId] = (instant.getNano() / 1_000) * 1_000;
+ }
+ }
+
+ private static class Decimal18Writer implements OrcValueWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal18Writer(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public void nonNullWrite(int rowId, DecimalData data, ColumnVector output) {
+ Preconditions.checkArgument(
+ scale == data.scale(),
+ "Cannot write value as decimal(%s,%s), wrong scale: %s",
+ precision,
+ scale,
+ data);
+ Preconditions.checkArgument(
+ data.precision() <= precision,
+ "Cannot write value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ data);
+
+ ((DecimalColumnVector) output)
+ .vector[rowId].setFromLongAndScale(data.toUnscaledLong(), data.scale());
+ }
+ }
+
+ private static class Decimal38Writer implements OrcValueWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal38Writer(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public void nonNullWrite(int rowId, DecimalData data, ColumnVector output) {
+ Preconditions.checkArgument(
+ scale == data.scale(),
+ "Cannot write value as decimal(%s,%s), wrong scale: %s",
+ precision,
+ scale,
+ data);
+ Preconditions.checkArgument(
+ data.precision() <= precision,
+ "Cannot write value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ data);
+
+ ((DecimalColumnVector) output)
+ .vector[rowId].set(HiveDecimal.create(data.toBigDecimal(), false));
+ }
+ }
+
+ static class ListWriter<T> implements OrcValueWriter<ArrayData> {
+ private final OrcValueWriter<T> elementWriter;
+ private final ArrayData.ElementGetter elementGetter;
+
+ ListWriter(OrcValueWriter<T> elementWriter, LogicalType elementType) {
+ this.elementWriter = elementWriter;
+ this.elementGetter = ArrayData.createElementGetter(elementType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void nonNullWrite(int rowId, ArrayData data, ColumnVector output) {
+ ListColumnVector cv = (ListColumnVector) output;
+ cv.lengths[rowId] = data.size();
+ cv.offsets[rowId] = cv.childCount;
+ cv.childCount = (int) (cv.childCount + cv.lengths[rowId]);
+ // make sure the child is big enough.
+ growColumnVector(cv.child, cv.childCount);
+
+ for (int e = 0; e < cv.lengths[rowId]; ++e) {
+ Object value = elementGetter.getElementOrNull(data, e);
+ elementWriter.write((int) (e + cv.offsets[rowId]), (T) value, cv.child);
+ }
+ }
+
+ @Override
+ public Stream<FieldMetrics<?>> metrics() {
+ return elementWriter.metrics();
+ }
+ }
+
+ static class MapWriter<K, V> implements OrcValueWriter<MapData> {
+ private final OrcValueWriter<K> keyWriter;
+ private final OrcValueWriter<V> valueWriter;
+ private final ArrayData.ElementGetter keyGetter;
+ private final ArrayData.ElementGetter valueGetter;
+
+ MapWriter(
+ OrcValueWriter<K> keyWriter,
+ OrcValueWriter<V> valueWriter,
+ LogicalType keyType,
+ LogicalType valueType) {
+ this.keyWriter = keyWriter;
+ this.valueWriter = valueWriter;
+ this.keyGetter = ArrayData.createElementGetter(keyType);
+ this.valueGetter = ArrayData.createElementGetter(valueType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void nonNullWrite(int rowId, MapData data, ColumnVector output) {
+ MapColumnVector cv = (MapColumnVector) output;
+ ArrayData keyArray = data.keyArray();
+ ArrayData valArray = data.valueArray();
+
+ // record the length and start of the list elements
+ cv.lengths[rowId] = data.size();
+ cv.offsets[rowId] = cv.childCount;
+ cv.childCount = (int) (cv.childCount + cv.lengths[rowId]);
+ // make sure the child is big enough
+ growColumnVector(cv.keys, cv.childCount);
+ growColumnVector(cv.values, cv.childCount);
+ // Add each element
+ for (int e = 0; e < cv.lengths[rowId]; ++e) {
+ int pos = (int) (e + cv.offsets[rowId]);
+ keyWriter.write(pos, (K) keyGetter.getElementOrNull(keyArray, e), cv.keys);
+ valueWriter.write(pos, (V) valueGetter.getElementOrNull(valArray, e), cv.values);
+ }
+ }
+
+ @Override
+ public Stream<FieldMetrics<?>> metrics() {
+ return Stream.concat(keyWriter.metrics(), valueWriter.metrics());
+ }
+ }
+
+ static class RowDataWriter extends GenericOrcWriters.StructWriter<RowData> {
+ private final List<RowData.FieldGetter> fieldGetters;
+
+ RowDataWriter(List<OrcValueWriter<?>> writers, List<LogicalType> types) {
+ super(writers);
+
+ this.fieldGetters = Lists.newArrayListWithExpectedSize(types.size());
+ for (int i = 0; i < types.size(); i++) {
+ fieldGetters.add(RowData.createFieldGetter(types.get(i), i));
+ }
+ }
+
+ @Override
+ protected Object get(RowData struct, int index) {
+ return fieldGetters.get(index).getFieldOrNull(struct);
+ }
+ }
+
+ private static void growColumnVector(ColumnVector cv, int requestedSize) {
+ if (cv.isNull.length < requestedSize) {
+ // Use growth factor of 3 to avoid frequent array allocations
+ cv.ensureSize(requestedSize * 3, true);
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java
new file mode 100644
index 0000000000..ab7b1174c9
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java
@@ -0,0 +1,832 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericArrayData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RawValueData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.iceberg.MetadataColumns;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.parquet.ParquetSchemaUtil;
+import org.apache.iceberg.parquet.ParquetValueReader;
+import org.apache.iceberg.parquet.ParquetValueReaders;
+import org.apache.iceberg.parquet.TypeWithSchemaVisitor;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ArrayUtil;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+public class FlinkParquetReaders {
+ private FlinkParquetReaders() {}
+
+ public static ParquetValueReader<RowData> buildReader(
+ Schema expectedSchema, MessageType fileSchema) {
+ return buildReader(expectedSchema, fileSchema, ImmutableMap.of());
+ }
+
+ @SuppressWarnings("unchecked")
+ public static ParquetValueReader<RowData> buildReader(
+ Schema expectedSchema, MessageType fileSchema, Map<Integer, ?> idToConstant) {
+ return (ParquetValueReader<RowData>)
+ TypeWithSchemaVisitor.visit(
+ expectedSchema.asStruct(), fileSchema, new ReadBuilder(fileSchema, idToConstant));
+ }
+
+ private static class ReadBuilder extends TypeWithSchemaVisitor<ParquetValueReader<?>> {
+ private final MessageType type;
+ private final Map<Integer, ?> idToConstant;
+
+ ReadBuilder(MessageType type, Map<Integer, ?> idToConstant) {
+ this.type = type;
+ this.idToConstant = idToConstant;
+ }
+
+ @Override
+ public ParquetValueReader<RowData> message(
+ Types.StructType expected, MessageType message, List<ParquetValueReader<?>> fieldReaders) {
+ return struct(expected, message.asGroupType(), fieldReaders);
+ }
+
+ @Override
+ public ParquetValueReader<RowData> struct(
+ Types.StructType expected, GroupType struct, List<ParquetValueReader<?>> fieldReaders) {
+ // match the expected struct's order
+ Map<Integer, ParquetValueReader<?>> readersById = Maps.newHashMap();
+ Map<Integer, Type> typesById = Maps.newHashMap();
+ Map<Integer, Integer> maxDefinitionLevelsById = Maps.newHashMap();
+ List<Type> fields = struct.getFields();
+ for (int i = 0; i < fields.size(); i += 1) {
+ Type fieldType = fields.get(i);
+ if (fieldReaders.get(i) != null) {
+ int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1;
+ if (fieldType.getId() != null) {
+ int id = fieldType.getId().intValue();
+ readersById.put(id, ParquetValueReaders.option(fieldType, fieldD, fieldReaders.get(i)));
+ typesById.put(id, fieldType);
+ if (idToConstant.containsKey(id)) {
+ maxDefinitionLevelsById.put(id, fieldD);
+ }
+ }
+ }
+ }
+
+ List<Types.NestedField> expectedFields =
+ expected != null ? expected.fields() : ImmutableList.of();
+ List<ParquetValueReader<?>> reorderedFields =
+ Lists.newArrayListWithExpectedSize(expectedFields.size());
+ List<Type> types = Lists.newArrayListWithExpectedSize(expectedFields.size());
+ // Defaulting to parent max definition level
+ int defaultMaxDefinitionLevel = type.getMaxDefinitionLevel(currentPath());
+ for (Types.NestedField field : expectedFields) {
+ int id = field.fieldId();
+ if (idToConstant.containsKey(id)) {
+ // containsKey is used because the constant may be null
+ int fieldMaxDefinitionLevel =
+ maxDefinitionLevelsById.getOrDefault(id, defaultMaxDefinitionLevel);
+ reorderedFields.add(
+ ParquetValueReaders.constant(idToConstant.get(id), fieldMaxDefinitionLevel));
+ types.add(null);
+ } else if (id == MetadataColumns.ROW_POSITION.fieldId()) {
+ reorderedFields.add(ParquetValueReaders.position());
+ types.add(null);
+ } else if (id == MetadataColumns.IS_DELETED.fieldId()) {
+ reorderedFields.add(ParquetValueReaders.constant(false));
+ types.add(null);
+ } else {
+ ParquetValueReader<?> reader = readersById.get(id);
+ if (reader != null) {
+ reorderedFields.add(reader);
+ types.add(typesById.get(id));
+ } else {
+ reorderedFields.add(ParquetValueReaders.nulls());
+ types.add(null);
+ }
+ }
+ }
+
+ return new RowDataReader(types, reorderedFields);
+ }
+
+ @Override
+ public ParquetValueReader<?> list(
+ Types.ListType expectedList, GroupType array, ParquetValueReader<?> elementReader) {
+ if (expectedList == null) {
+ return null;
+ }
+
+ String[] repeatedPath = currentPath();
+
+ int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
+ int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;
+
+ Type elementType = ParquetSchemaUtil.determineListElementType(array);
+ int elementD = type.getMaxDefinitionLevel(path(elementType.getName())) - 1;
+
+ return new ArrayReader<>(
+ repeatedD, repeatedR, ParquetValueReaders.option(elementType, elementD, elementReader));
+ }
+
+ @Override
+ public ParquetValueReader<?> map(
+ Types.MapType expectedMap,
+ GroupType map,
+ ParquetValueReader<?> keyReader,
+ ParquetValueReader<?> valueReader) {
+ if (expectedMap == null) {
+ return null;
+ }
+
+ GroupType repeatedKeyValue = map.getFields().get(0).asGroupType();
+ String[] repeatedPath = currentPath();
+
+ int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
+ int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;
+
+ Type keyType = repeatedKeyValue.getType(0);
+ int keyD = type.getMaxDefinitionLevel(path(keyType.getName())) - 1;
+ Type valueType = repeatedKeyValue.getType(1);
+ int valueD = type.getMaxDefinitionLevel(path(valueType.getName())) - 1;
+
+ return new MapReader<>(
+ repeatedD,
+ repeatedR,
+ ParquetValueReaders.option(keyType, keyD, keyReader),
+ ParquetValueReaders.option(valueType, valueD, valueReader));
+ }
+
+ @Override
+ @SuppressWarnings("CyclomaticComplexity")
+ public ParquetValueReader<?> primitive(
+ org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) {
+ if (expected == null) {
+ return null;
+ }
+
+ ColumnDescriptor desc = type.getColumnDescription(currentPath());
+
+ if (primitive.getOriginalType() != null) {
+ switch (primitive.getOriginalType()) {
+ case ENUM:
+ case JSON:
+ case UTF8:
+ return new StringReader(desc);
+ case INT_8:
+ case INT_16:
+ case INT_32:
+ if (expected.typeId() == Types.LongType.get().typeId()) {
+ return new ParquetValueReaders.IntAsLongReader(desc);
+ } else {
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ }
+ case TIME_MICROS:
+ return new LossyMicrosToMillisTimeReader(desc);
+ case TIME_MILLIS:
+ return new MillisTimeReader(desc);
+ case DATE:
+ case INT_64:
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ case TIMESTAMP_MICROS:
+ if (((Types.TimestampType) expected).shouldAdjustToUTC()) {
+ return new MicrosToTimestampTzReader(desc);
+ } else {
+ return new MicrosToTimestampReader(desc);
+ }
+ case TIMESTAMP_MILLIS:
+ if (((Types.TimestampType) expected).shouldAdjustToUTC()) {
+ return new MillisToTimestampTzReader(desc);
+ } else {
+ return new MillisToTimestampReader(desc);
+ }
+ case DECIMAL:
+ DecimalLogicalTypeAnnotation decimal =
+ (DecimalLogicalTypeAnnotation) primitive.getLogicalTypeAnnotation();
+ switch (primitive.getPrimitiveTypeName()) {
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ return new BinaryDecimalReader(desc, decimal.getPrecision(), decimal.getScale());
+ case INT64:
+ return new LongDecimalReader(desc, decimal.getPrecision(), decimal.getScale());
+ case INT32:
+ return new IntegerDecimalReader(desc, decimal.getPrecision(), decimal.getScale());
+ default:
+ throw new UnsupportedOperationException(
+ "Unsupported base type for decimal: " + primitive.getPrimitiveTypeName());
+ }
+ case BSON:
+ return new ParquetValueReaders.ByteArrayReader(desc);
+ default:
+ throw new UnsupportedOperationException(
+ "Unsupported logical type: " + primitive.getOriginalType());
+ }
+ }
+
+ switch (primitive.getPrimitiveTypeName()) {
+ case FIXED_LEN_BYTE_ARRAY:
+ case BINARY:
+ return new ParquetValueReaders.ByteArrayReader(desc);
+ case INT32:
+ if (expected.typeId() == org.apache.iceberg.types.Type.TypeID.LONG) {
+ return new ParquetValueReaders.IntAsLongReader(desc);
+ } else {
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ }
+ case FLOAT:
+ if (expected.typeId() == org.apache.iceberg.types.Type.TypeID.DOUBLE) {
+ return new ParquetValueReaders.FloatAsDoubleReader(desc);
+ } else {
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ }
+ case BOOLEAN:
+ case INT64:
+ case DOUBLE:
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + primitive);
+ }
+ }
+ }
+
+ private static class BinaryDecimalReader
+ extends ParquetValueReaders.PrimitiveReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ BinaryDecimalReader(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData read(DecimalData ignored) {
+ Binary binary = column.nextBinary();
+ BigDecimal bigDecimal = new BigDecimal(new BigInteger(binary.getBytes()), scale);
+ // TODO: need a unit test to write-read-validate decimal via FlinkParquetWrite/Reader
+ return DecimalData.fromBigDecimal(bigDecimal, precision, scale);
+ }
+ }
+
+ private static class IntegerDecimalReader
+ extends ParquetValueReaders.PrimitiveReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ IntegerDecimalReader(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData read(DecimalData ignored) {
+ return DecimalData.fromUnscaledLong(column.nextInteger(), precision, scale);
+ }
+ }
+
+ private static class LongDecimalReader extends ParquetValueReaders.PrimitiveReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ LongDecimalReader(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData read(DecimalData ignored) {
+ return DecimalData.fromUnscaledLong(column.nextLong(), precision, scale);
+ }
+ }
+
+ private static class MicrosToTimestampTzReader
+ extends ParquetValueReaders.UnboxedReader<TimestampData> {
+ MicrosToTimestampTzReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public TimestampData read(TimestampData ignored) {
+ long value = readLong();
+ return TimestampData.fromLocalDateTime(
+ Instant.ofEpochSecond(
+ Math.floorDiv(value, 1000_000), Math.floorMod(value, 1000_000) * 1000)
+ .atOffset(ZoneOffset.UTC)
+ .toLocalDateTime());
+ }
+
+ @Override
+ public long readLong() {
+ return column.nextLong();
+ }
+ }
+
+ private static class MicrosToTimestampReader
+ extends ParquetValueReaders.UnboxedReader<TimestampData> {
+ MicrosToTimestampReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public TimestampData read(TimestampData ignored) {
+ long value = readLong();
+ return TimestampData.fromInstant(
+ Instant.ofEpochSecond(
+ Math.floorDiv(value, 1000_000), Math.floorMod(value, 1000_000) * 1000));
+ }
+
+ @Override
+ public long readLong() {
+ return column.nextLong();
+ }
+ }
+
+ private static class MillisToTimestampReader
+ extends ParquetValueReaders.UnboxedReader<TimestampData> {
+ MillisToTimestampReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public TimestampData read(TimestampData ignored) {
+ long millis = readLong();
+ return TimestampData.fromEpochMillis(millis);
+ }
+
+ @Override
+ public long readLong() {
+ return column.nextLong();
+ }
+ }
+
+ private static class MillisToTimestampTzReader
+ extends ParquetValueReaders.UnboxedReader<TimestampData> {
+ MillisToTimestampTzReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public TimestampData read(TimestampData ignored) {
+ long millis = readLong();
+ return TimestampData.fromLocalDateTime(
+ Instant.ofEpochMilli(millis).atOffset(ZoneOffset.UTC).toLocalDateTime());
+ }
+
+ @Override
+ public long readLong() {
+ return column.nextLong();
+ }
+ }
+
+ private static class StringReader extends ParquetValueReaders.PrimitiveReader<StringData> {
+ StringReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public StringData read(StringData ignored) {
+ Binary binary = column.nextBinary();
+ ByteBuffer buffer = binary.toByteBuffer();
+ if (buffer.hasArray()) {
+ return StringData.fromBytes(
+ buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
+ } else {
+ return StringData.fromBytes(binary.getBytes());
+ }
+ }
+ }
+
+ private static class LossyMicrosToMillisTimeReader
+ extends ParquetValueReaders.PrimitiveReader<Integer> {
+ LossyMicrosToMillisTimeReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public Integer read(Integer reuse) {
+ // Discard microseconds since Flink uses millisecond unit for TIME type.
+ return (int) Math.floorDiv(column.nextLong(), 1000);
+ }
+ }
+
+ private static class MillisTimeReader extends ParquetValueReaders.PrimitiveReader<Integer> {
+ MillisTimeReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public Integer read(Integer reuse) {
+ return (int) column.nextLong();
+ }
+ }
+
+ private static class ArrayReader<E>
+ extends ParquetValueReaders.RepeatedReader<ArrayData, ReusableArrayData, E> {
+ private int readPos = 0;
+ private int writePos = 0;
+
+ ArrayReader(int definitionLevel, int repetitionLevel, ParquetValueReader<E> reader) {
+ super(definitionLevel, repetitionLevel, reader);
+ }
+
+ @Override
+ protected ReusableArrayData newListData(ArrayData reuse) {
+ this.readPos = 0;
+ this.writePos = 0;
+
+ if (reuse instanceof ReusableArrayData) {
+ return (ReusableArrayData) reuse;
+ } else {
+ return new ReusableArrayData();
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected E getElement(ReusableArrayData list) {
+ E value = null;
+ if (readPos < list.capacity()) {
+ value = (E) list.values[readPos];
+ }
+
+ readPos += 1;
+
+ return value;
+ }
+
+ @Override
+ protected void addElement(ReusableArrayData reused, E element) {
+ if (writePos >= reused.capacity()) {
+ reused.grow();
+ }
+
+ reused.values[writePos] = element;
+
+ writePos += 1;
+ }
+
+ @Override
+ protected ArrayData buildList(ReusableArrayData list) {
+ // Since ReusableArrayData is not accepted by Flink, use GenericArrayData temporarily to walk
+ // around it.
+ // Revert this to use ReusableArrayData once it is fixed in Flink.
+ // For your reference, https://issues.apache.org/jira/browse/FLINK-25238.
+ return new GenericArrayData(Arrays.copyOf(list.values, writePos));
+ }
+ }
+
+ private static class MapReader<K, V>
+ extends ParquetValueReaders.RepeatedKeyValueReader<MapData, ReusableMapData, K, V> {
+ private int readPos = 0;
+ private int writePos = 0;
+
+ private final ParquetValueReaders.ReusableEntry<K, V> entry =
+ new ParquetValueReaders.ReusableEntry<>();
+ private final ParquetValueReaders.ReusableEntry<K, V> nullEntry =
+ new ParquetValueReaders.ReusableEntry<>();
+
+ MapReader(
+ int definitionLevel,
+ int repetitionLevel,
+ ParquetValueReader<K> keyReader,
+ ParquetValueReader<V> valueReader) {
+ super(definitionLevel, repetitionLevel, keyReader, valueReader);
+ }
+
+ @Override
+ protected ReusableMapData newMapData(MapData reuse) {
+ this.readPos = 0;
+ this.writePos = 0;
+
+ if (reuse instanceof ReusableMapData) {
+ return (ReusableMapData) reuse;
+ } else {
+ return new ReusableMapData();
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected Map.Entry<K, V> getPair(ReusableMapData map) {
+ Map.Entry<K, V> kv = nullEntry;
+ if (readPos < map.capacity()) {
+ entry.set((K) map.keys.values[readPos], (V) map.values.values[readPos]);
+ kv = entry;
+ }
+
+ readPos += 1;
+
+ return kv;
+ }
+
+ @Override
+ protected void addPair(ReusableMapData map, K key, V value) {
+ if (writePos >= map.capacity()) {
+ map.grow();
+ }
+
+ map.keys.values[writePos] = key;
+ map.values.values[writePos] = value;
+
+ writePos += 1;
+ }
+
+ @Override
+ protected MapData buildMap(ReusableMapData map) {
+ map.setNumElements(writePos);
+ return map;
+ }
+ }
+
+ private static class RowDataReader
+ extends ParquetValueReaders.StructReader<RowData, GenericRowData> {
+ private final int numFields;
+
+ RowDataReader(List<Type> types, List<ParquetValueReader<?>> readers) {
+ super(types, readers);
+ this.numFields = readers.size();
+ }
+
+ @Override
+ protected GenericRowData newStructData(RowData reuse) {
+ if (reuse instanceof GenericRowData) {
+ return (GenericRowData) reuse;
+ } else {
+ return new GenericRowData(numFields);
+ }
+ }
+
+ @Override
+ protected Object getField(GenericRowData intermediate, int pos) {
+ return intermediate.getField(pos);
+ }
+
+ @Override
+ protected RowData buildStruct(GenericRowData struct) {
+ return struct;
+ }
+
+ @Override
+ protected void set(GenericRowData row, int pos, Object value) {
+ row.setField(pos, value);
+ }
+
+ @Override
+ protected void setNull(GenericRowData row, int pos) {
+ row.setField(pos, null);
+ }
+
+ @Override
+ protected void setBoolean(GenericRowData row, int pos, boolean value) {
+ row.setField(pos, value);
+ }
+
+ @Override
+ protected void setInteger(GenericRowData row, int pos, int value) {
+ row.setField(pos, value);
+ }
+
+ @Override
+ protected void setLong(GenericRowData row, int pos, long value) {
+ row.setField(pos, value);
+ }
+
+ @Override
+ protected void setFloat(GenericRowData row, int pos, float value) {
+ row.setField(pos, value);
+ }
+
+ @Override
+ protected void setDouble(GenericRowData row, int pos, double value) {
+ row.setField(pos, value);
+ }
+ }
+
+ private static class ReusableMapData implements MapData {
+ private final ReusableArrayData keys;
+ private final ReusableArrayData values;
+
+ private int numElements;
+
+ private ReusableMapData() {
+ this.keys = new ReusableArrayData();
+ this.values = new ReusableArrayData();
+ }
+
+ private void grow() {
+ keys.grow();
+ values.grow();
+ }
+
+ private int capacity() {
+ return keys.capacity();
+ }
+
+ public void setNumElements(int numElements) {
+ this.numElements = numElements;
+ keys.setNumElements(numElements);
+ values.setNumElements(numElements);
+ }
+
+ @Override
+ public int size() {
+ return numElements;
+ }
+
+ @Override
+ public ReusableArrayData keyArray() {
+ return keys;
+ }
+
+ @Override
+ public ReusableArrayData valueArray() {
+ return values;
+ }
+ }
+
+ private static class ReusableArrayData implements ArrayData {
+ private static final Object[] EMPTY = new Object[0];
+
+ private Object[] values = EMPTY;
+ private int numElements = 0;
+
+ private void grow() {
+ if (values.length == 0) {
+ this.values = new Object[20];
+ } else {
+ Object[] old = values;
+ this.values = new Object[old.length << 1];
+ // copy the old array in case it has values that can be reused
+ System.arraycopy(old, 0, values, 0, old.length);
+ }
+ }
+
+ private int capacity() {
+ return values.length;
+ }
+
+ public void setNumElements(int numElements) {
+ this.numElements = numElements;
+ }
+
+ @Override
+ public int size() {
+ return numElements;
+ }
+
+ @Override
+ public boolean isNullAt(int ordinal) {
+ return null == values[ordinal];
+ }
+
+ @Override
+ public boolean getBoolean(int ordinal) {
+ return (boolean) values[ordinal];
+ }
+
+ @Override
+ public byte getByte(int ordinal) {
+ return (byte) values[ordinal];
+ }
+
+ @Override
+ public short getShort(int ordinal) {
+ return (short) values[ordinal];
+ }
+
+ @Override
+ public int getInt(int ordinal) {
+ return (int) values[ordinal];
+ }
+
+ @Override
+ public long getLong(int ordinal) {
+ return (long) values[ordinal];
+ }
+
+ @Override
+ public float getFloat(int ordinal) {
+ return (float) values[ordinal];
+ }
+
+ @Override
+ public double getDouble(int ordinal) {
+ return (double) values[ordinal];
+ }
+
+ @Override
+ public StringData getString(int pos) {
+ return (StringData) values[pos];
+ }
+
+ @Override
+ public DecimalData getDecimal(int pos, int precision, int scale) {
+ return (DecimalData) values[pos];
+ }
+
+ @Override
+ public TimestampData getTimestamp(int pos, int precision) {
+ return (TimestampData) values[pos];
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> RawValueData<T> getRawValue(int pos) {
+ return (RawValueData<T>) values[pos];
+ }
+
+ @Override
+ public byte[] getBinary(int ordinal) {
+ return (byte[]) values[ordinal];
+ }
+
+ @Override
+ public ArrayData getArray(int ordinal) {
+ return (ArrayData) values[ordinal];
+ }
+
+ @Override
+ public MapData getMap(int ordinal) {
+ return (MapData) values[ordinal];
+ }
+
+ @Override
+ public RowData getRow(int pos, int numFields) {
+ return (RowData) values[pos];
+ }
+
+ @Override
+ public boolean[] toBooleanArray() {
+ return ArrayUtil.toPrimitive((Boolean[]) values);
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ return ArrayUtil.toPrimitive((Byte[]) values);
+ }
+
+ @Override
+ public short[] toShortArray() {
+ return ArrayUtil.toPrimitive((Short[]) values);
+ }
+
+ @Override
+ public int[] toIntArray() {
+ return ArrayUtil.toPrimitive((Integer[]) values);
+ }
+
+ @Override
+ public long[] toLongArray() {
+ return ArrayUtil.toPrimitive((Long[]) values);
+ }
+
+ @Override
+ public float[] toFloatArray() {
+ return ArrayUtil.toPrimitive((Float[]) values);
+ }
+
+ @Override
+ public double[] toDoubleArray() {
+ return ArrayUtil.toPrimitive((Double[]) values);
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
new file mode 100644
index 0000000000..db4f1730a1
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
@@ -0,0 +1,504 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.RowType.RowField;
+import org.apache.flink.table.types.logical.SmallIntType;
+import org.apache.flink.table.types.logical.TinyIntType;
+import org.apache.iceberg.parquet.ParquetValueReaders;
+import org.apache.iceberg.parquet.ParquetValueWriter;
+import org.apache.iceberg.parquet.ParquetValueWriters;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.util.DecimalUtil;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+public class FlinkParquetWriters {
+ private FlinkParquetWriters() {}
+
+ @SuppressWarnings("unchecked")
+ public static <T> ParquetValueWriter<T> buildWriter(LogicalType schema, MessageType type) {
+ return (ParquetValueWriter<T>)
+ ParquetWithFlinkSchemaVisitor.visit(schema, type, new WriteBuilder(type));
+ }
+
+ private static class WriteBuilder extends ParquetWithFlinkSchemaVisitor<ParquetValueWriter<?>> {
+ private final MessageType type;
+
+ WriteBuilder(MessageType type) {
+ this.type = type;
+ }
+
+ @Override
+ public ParquetValueWriter<?> message(
+ RowType sStruct, MessageType message, List<ParquetValueWriter<?>> fields) {
+ return struct(sStruct, message.asGroupType(), fields);
+ }
+
+ @Override
+ public ParquetValueWriter<?> struct(
+ RowType sStruct, GroupType struct, List<ParquetValueWriter<?>> fieldWriters) {
+ List<Type> fields = struct.getFields();
+ List<RowField> flinkFields = sStruct.getFields();
+ List<ParquetValueWriter<?>> writers = Lists.newArrayListWithExpectedSize(fieldWriters.size());
+ List<LogicalType> flinkTypes = Lists.newArrayList();
+ for (int i = 0; i < fields.size(); i += 1) {
+ writers.add(newOption(struct.getType(i), fieldWriters.get(i)));
+ flinkTypes.add(flinkFields.get(i).getType());
+ }
+
+ return new RowDataWriter(writers, flinkTypes);
+ }
+
+ @Override
+ public ParquetValueWriter<?> list(
+ ArrayType sArray, GroupType array, ParquetValueWriter<?> elementWriter) {
+ GroupType repeated = array.getFields().get(0).asGroupType();
+ String[] repeatedPath = currentPath();
+
+ int repeatedD = type.getMaxDefinitionLevel(repeatedPath);
+ int repeatedR = type.getMaxRepetitionLevel(repeatedPath);
+
+ return new ArrayDataWriter<>(
+ repeatedD,
+ repeatedR,
+ newOption(repeated.getType(0), elementWriter),
+ sArray.getElementType());
+ }
+
+ @Override
+ public ParquetValueWriter<?> map(
+ MapType sMap,
+ GroupType map,
+ ParquetValueWriter<?> keyWriter,
+ ParquetValueWriter<?> valueWriter) {
+ GroupType repeatedKeyValue = map.getFields().get(0).asGroupType();
+ String[] repeatedPath = currentPath();
+
+ int repeatedD = type.getMaxDefinitionLevel(repeatedPath);
+ int repeatedR = type.getMaxRepetitionLevel(repeatedPath);
+
+ return new MapDataWriter<>(
+ repeatedD,
+ repeatedR,
+ newOption(repeatedKeyValue.getType(0), keyWriter),
+ newOption(repeatedKeyValue.getType(1), valueWriter),
+ sMap.getKeyType(),
+ sMap.getValueType());
+ }
+
+ private ParquetValueWriter<?> newOption(Type fieldType, ParquetValueWriter<?> writer) {
+ int maxD = type.getMaxDefinitionLevel(path(fieldType.getName()));
+ return ParquetValueWriters.option(fieldType, maxD, writer);
+ }
+
+ @Override
+ public ParquetValueWriter<?> primitive(LogicalType fType, PrimitiveType primitive) {
+ ColumnDescriptor desc = type.getColumnDescription(currentPath());
+
+ if (primitive.getOriginalType() != null) {
+ switch (primitive.getOriginalType()) {
+ case ENUM:
+ case JSON:
+ case UTF8:
+ return strings(desc);
+ case DATE:
+ case INT_8:
+ case INT_16:
+ case INT_32:
+ return ints(fType, desc);
+ case INT_64:
+ return ParquetValueWriters.longs(desc);
+ case TIME_MICROS:
+ return timeMicros(desc);
+ case TIMESTAMP_MICROS:
+ return timestamps(desc);
+ case DECIMAL:
+ DecimalLogicalTypeAnnotation decimal =
+ (DecimalLogicalTypeAnnotation) primitive.getLogicalTypeAnnotation();
+ switch (primitive.getPrimitiveTypeName()) {
+ case INT32:
+ return decimalAsInteger(desc, decimal.getPrecision(), decimal.getScale());
+ case INT64:
+ return decimalAsLong(desc, decimal.getPrecision(), decimal.getScale());
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ return decimalAsFixed(desc, decimal.getPrecision(), decimal.getScale());
+ default:
+ throw new UnsupportedOperationException(
+ "Unsupported base type for decimal: " + primitive.getPrimitiveTypeName());
+ }
+ case BSON:
+ return byteArrays(desc);
+ default:
+ throw new UnsupportedOperationException(
+ "Unsupported logical type: " + primitive.getOriginalType());
+ }
+ }
+
+ switch (primitive.getPrimitiveTypeName()) {
+ case FIXED_LEN_BYTE_ARRAY:
+ case BINARY:
+ return byteArrays(desc);
+ case BOOLEAN:
+ return ParquetValueWriters.booleans(desc);
+ case INT32:
+ return ints(fType, desc);
+ case INT64:
+ return ParquetValueWriters.longs(desc);
+ case FLOAT:
+ return ParquetValueWriters.floats(desc);
+ case DOUBLE:
+ return ParquetValueWriters.doubles(desc);
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + primitive);
+ }
+ }
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<?> ints(
+ LogicalType type, ColumnDescriptor desc) {
+ if (type instanceof TinyIntType) {
+ return ParquetValueWriters.tinyints(desc);
+ } else if (type instanceof SmallIntType) {
+ return ParquetValueWriters.shorts(desc);
+ }
+ return ParquetValueWriters.ints(desc);
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<StringData> strings(ColumnDescriptor desc) {
+ return new StringDataWriter(desc);
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<Integer> timeMicros(ColumnDescriptor desc) {
+ return new TimeMicrosWriter(desc);
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<DecimalData> decimalAsInteger(
+ ColumnDescriptor desc, int precision, int scale) {
+ Preconditions.checkArgument(
+ precision <= 9,
+ "Cannot write decimal value as integer with precision larger than 9,"
+ + " wrong precision %s",
+ precision);
+ return new IntegerDecimalWriter(desc, precision, scale);
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<DecimalData> decimalAsLong(
+ ColumnDescriptor desc, int precision, int scale) {
+ Preconditions.checkArgument(
+ precision <= 18,
+ "Cannot write decimal value as long with precision larger than 18, "
+ + " wrong precision %s",
+ precision);
+ return new LongDecimalWriter(desc, precision, scale);
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<DecimalData> decimalAsFixed(
+ ColumnDescriptor desc, int precision, int scale) {
+ return new FixedDecimalWriter(desc, precision, scale);
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<TimestampData> timestamps(
+ ColumnDescriptor desc) {
+ return new TimestampDataWriter(desc);
+ }
+
+ private static ParquetValueWriters.PrimitiveWriter<byte[]> byteArrays(ColumnDescriptor desc) {
+ return new ByteArrayWriter(desc);
+ }
+
+ private static class StringDataWriter extends ParquetValueWriters.PrimitiveWriter<StringData> {
+ private StringDataWriter(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public void write(int repetitionLevel, StringData value) {
+ column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(value.toBytes()));
+ }
+ }
+
+ private static class TimeMicrosWriter extends ParquetValueWriters.PrimitiveWriter<Integer> {
+ private TimeMicrosWriter(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public void write(int repetitionLevel, Integer value) {
+ long micros = value.longValue() * 1000;
+ column.writeLong(repetitionLevel, micros);
+ }
+ }
+
+ private static class IntegerDecimalWriter
+ extends ParquetValueWriters.PrimitiveWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ private IntegerDecimalWriter(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public void write(int repetitionLevel, DecimalData decimal) {
+ Preconditions.checkArgument(
+ decimal.scale() == scale,
+ "Cannot write value as decimal(%s,%s), wrong scale: %s",
+ precision,
+ scale,
+ decimal);
+ Preconditions.checkArgument(
+ decimal.precision() <= precision,
+ "Cannot write value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ decimal);
+
+ column.writeInteger(repetitionLevel, (int) decimal.toUnscaledLong());
+ }
+ }
+
+ private static class LongDecimalWriter extends ParquetValueWriters.PrimitiveWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ private LongDecimalWriter(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public void write(int repetitionLevel, DecimalData decimal) {
+ Preconditions.checkArgument(
+ decimal.scale() == scale,
+ "Cannot write value as decimal(%s,%s), wrong scale: %s",
+ precision,
+ scale,
+ decimal);
+ Preconditions.checkArgument(
+ decimal.precision() <= precision,
+ "Cannot write value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ decimal);
+
+ column.writeLong(repetitionLevel, decimal.toUnscaledLong());
+ }
+ }
+
+ private static class FixedDecimalWriter extends ParquetValueWriters.PrimitiveWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+ private final ThreadLocal<byte[]> bytes;
+
+ private FixedDecimalWriter(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ this.bytes =
+ ThreadLocal.withInitial(() -> new byte[TypeUtil.decimalRequiredBytes(precision)]);
+ }
+
+ @Override
+ public void write(int repetitionLevel, DecimalData decimal) {
+ byte[] binary =
+ DecimalUtil.toReusedFixLengthBytes(precision, scale, decimal.toBigDecimal(), bytes.get());
+ column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(binary));
+ }
+ }
+
+ private static class TimestampDataWriter
+ extends ParquetValueWriters.PrimitiveWriter<TimestampData> {
+ private TimestampDataWriter(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public void write(int repetitionLevel, TimestampData value) {
+ column.writeLong(
+ repetitionLevel, value.getMillisecond() * 1000 + value.getNanoOfMillisecond() / 1000);
+ }
+ }
+
+ private static class ByteArrayWriter extends ParquetValueWriters.PrimitiveWriter<byte[]> {
+ private ByteArrayWriter(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public void write(int repetitionLevel, byte[] bytes) {
+ column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(bytes));
+ }
+ }
+
+ private static class ArrayDataWriter<E> extends ParquetValueWriters.RepeatedWriter<ArrayData, E> {
+ private final LogicalType elementType;
+
+ private ArrayDataWriter(
+ int definitionLevel,
+ int repetitionLevel,
+ ParquetValueWriter<E> writer,
+ LogicalType elementType) {
+ super(definitionLevel, repetitionLevel, writer);
+ this.elementType = elementType;
+ }
+
+ @Override
+ protected Iterator<E> elements(ArrayData list) {
+ return new ElementIterator<>(list);
+ }
+
+ private class ElementIterator<E> implements Iterator<E> {
+ private final int size;
+ private final ArrayData list;
+ private final ArrayData.ElementGetter getter;
+ private int index;
+
+ private ElementIterator(ArrayData list) {
+ this.list = list;
+ size = list.size();
+ getter = ArrayData.createElementGetter(elementType);
+ index = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return index != size;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public E next() {
+ if (index >= size) {
+ throw new NoSuchElementException();
+ }
+
+ E element = (E) getter.getElementOrNull(list, index);
+ index += 1;
+
+ return element;
+ }
+ }
+ }
+
+ private static class MapDataWriter<K, V>
+ extends ParquetValueWriters.RepeatedKeyValueWriter<MapData, K, V> {
+ private final LogicalType keyType;
+ private final LogicalType valueType;
+
+ private MapDataWriter(
+ int definitionLevel,
+ int repetitionLevel,
+ ParquetValueWriter<K> keyWriter,
+ ParquetValueWriter<V> valueWriter,
+ LogicalType keyType,
+ LogicalType valueType) {
+ super(definitionLevel, repetitionLevel, keyWriter, valueWriter);
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+ @Override
+ protected Iterator<Map.Entry<K, V>> pairs(MapData map) {
+ return new EntryIterator<>(map);
+ }
+
+ private class EntryIterator<K, V> implements Iterator<Map.Entry<K, V>> {
+ private final int size;
+ private final ArrayData keys;
+ private final ArrayData values;
+ private final ParquetValueReaders.ReusableEntry<K, V> entry;
+ private final ArrayData.ElementGetter keyGetter;
+ private final ArrayData.ElementGetter valueGetter;
+ private int index;
+
+ private EntryIterator(MapData map) {
+ size = map.size();
+ keys = map.keyArray();
+ values = map.valueArray();
+ entry = new ParquetValueReaders.ReusableEntry<>();
+ keyGetter = ArrayData.createElementGetter(keyType);
+ valueGetter = ArrayData.createElementGetter(valueType);
+ index = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return index != size;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Map.Entry<K, V> next() {
+ if (index >= size) {
+ throw new NoSuchElementException();
+ }
+
+ entry.set(
+ (K) keyGetter.getElementOrNull(keys, index),
+ (V) valueGetter.getElementOrNull(values, index));
+ index += 1;
+
+ return entry;
+ }
+ }
+ }
+
+ private static class RowDataWriter extends ParquetValueWriters.StructWriter<RowData> {
+ private final RowData.FieldGetter[] fieldGetter;
+
+ RowDataWriter(List<ParquetValueWriter<?>> writers, List<LogicalType> types) {
+ super(writers);
+ fieldGetter = new RowData.FieldGetter[types.size()];
+ for (int i = 0; i < types.size(); i += 1) {
+ fieldGetter[i] = RowData.createFieldGetter(types.get(i), i);
+ }
+ }
+
+ @Override
+ protected Object get(RowData struct, int index) {
+ return fieldGetter[index].getFieldOrNull(struct);
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkSchemaVisitor.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkSchemaVisitor.java
new file mode 100644
index 0000000000..ba4e1a7a7a
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkSchemaVisitor.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.List;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+abstract class FlinkSchemaVisitor<T> {
+
+ static <T> T visit(RowType flinkType, Schema schema, FlinkSchemaVisitor<T> visitor) {
+ return visit(flinkType, schema.asStruct(), visitor);
+ }
+
+ private static <T> T visit(LogicalType flinkType, Type iType, FlinkSchemaVisitor<T> visitor) {
+ switch (iType.typeId()) {
+ case STRUCT:
+ return visitRecord(flinkType, iType.asStructType(), visitor);
+
+ case MAP:
+ MapType mapType = (MapType) flinkType;
+ Types.MapType iMapType = iType.asMapType();
+ T key;
+ T value;
+
+ Types.NestedField keyField = iMapType.field(iMapType.keyId());
+ visitor.beforeMapKey(keyField);
+ try {
+ key = visit(mapType.getKeyType(), iMapType.keyType(), visitor);
+ } finally {
+ visitor.afterMapKey(keyField);
+ }
+
+ Types.NestedField valueField = iMapType.field(iMapType.valueId());
+ visitor.beforeMapValue(valueField);
+ try {
+ value = visit(mapType.getValueType(), iMapType.valueType(), visitor);
+ } finally {
+ visitor.afterMapValue(valueField);
+ }
+
+ return visitor.map(iMapType, key, value, mapType.getKeyType(), mapType.getValueType());
+
+ case LIST:
+ ArrayType listType = (ArrayType) flinkType;
+ Types.ListType iListType = iType.asListType();
+ T element;
+
+ Types.NestedField elementField = iListType.field(iListType.elementId());
+ visitor.beforeListElement(elementField);
+ try {
+ element = visit(listType.getElementType(), iListType.elementType(), visitor);
+ } finally {
+ visitor.afterListElement(elementField);
+ }
+
+ return visitor.list(iListType, element, listType.getElementType());
+
+ default:
+ return visitor.primitive(iType.asPrimitiveType(), flinkType);
+ }
+ }
+
+ private static <T> T visitRecord(
+ LogicalType flinkType, Types.StructType struct, FlinkSchemaVisitor<T> visitor) {
+ Preconditions.checkArgument(flinkType instanceof RowType, "%s is not a RowType.", flinkType);
+ RowType rowType = (RowType) flinkType;
+
+ int fieldSize = struct.fields().size();
+ List<T> results = Lists.newArrayListWithExpectedSize(fieldSize);
+ List<LogicalType> fieldTypes = Lists.newArrayListWithExpectedSize(fieldSize);
+ List<Types.NestedField> nestedFields = struct.fields();
+
+ for (int i = 0; i < fieldSize; i++) {
+ Types.NestedField iField = nestedFields.get(i);
+ int fieldIndex = rowType.getFieldIndex(iField.name());
+ Preconditions.checkArgument(
+ fieldIndex >= 0, "NestedField: %s is not found in flink RowType: %s", iField, rowType);
+
+ LogicalType fieldFlinkType = rowType.getTypeAt(fieldIndex);
+
+ fieldTypes.add(fieldFlinkType);
+
+ visitor.beforeField(iField);
+ try {
+ results.add(visit(fieldFlinkType, iField.type(), visitor));
+ } finally {
+ visitor.afterField(iField);
+ }
+ }
+
+ return visitor.record(struct, results, fieldTypes);
+ }
+
+ public T record(Types.StructType iStruct, List<T> results, List<LogicalType> fieldTypes) {
+ return null;
+ }
+
+ public T list(Types.ListType iList, T element, LogicalType elementType) {
+ return null;
+ }
+
+ public T map(Types.MapType iMap, T key, T value, LogicalType keyType, LogicalType valueType) {
+ return null;
+ }
+
+ public T primitive(Type.PrimitiveType iPrimitive, LogicalType flinkPrimitive) {
+ return null;
+ }
+
+ public void beforeField(Types.NestedField field) {}
+
+ public void afterField(Types.NestedField field) {}
+
+ public void beforeListElement(Types.NestedField elementField) {
+ beforeField(elementField);
+ }
+
+ public void afterListElement(Types.NestedField elementField) {
+ afterField(elementField);
+ }
+
+ public void beforeMapKey(Types.NestedField keyField) {
+ beforeField(keyField);
+ }
+
+ public void afterMapKey(Types.NestedField keyField) {
+ afterField(keyField);
+ }
+
+ public void beforeMapValue(Types.NestedField valueField) {
+ beforeField(valueField);
+ }
+
+ public void afterMapValue(Types.NestedField valueField) {
+ afterField(valueField);
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueReaders.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueReaders.java
new file mode 100644
index 0000000000..32f6c3a2cc
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueReaders.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.util.Utf8;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericArrayData;
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.iceberg.avro.ValueReader;
+import org.apache.iceberg.avro.ValueReaders;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+
+public class FlinkValueReaders {
+
+ private FlinkValueReaders() {}
+
+ static ValueReader<StringData> strings() {
+ return StringReader.INSTANCE;
+ }
+
+ static ValueReader<StringData> enums(List<String> symbols) {
+ return new EnumReader(symbols);
+ }
+
+ static ValueReader<byte[]> uuids() {
+ return ValueReaders.fixed(16);
+ }
+
+ static ValueReader<Integer> timeMicros() {
+ return TimeMicrosReader.INSTANCE;
+ }
+
+ static ValueReader<TimestampData> timestampMills() {
+ return TimestampMillsReader.INSTANCE;
+ }
+
+ static ValueReader<TimestampData> timestampMicros() {
+ return TimestampMicrosReader.INSTANCE;
+ }
+
+ static ValueReader<DecimalData> decimal(
+ ValueReader<byte[]> unscaledReader, int precision, int scale) {
+ return new DecimalReader(unscaledReader, precision, scale);
+ }
+
+ static ValueReader<ArrayData> array(ValueReader<?> elementReader) {
+ return new ArrayReader(elementReader);
+ }
+
+ static ValueReader<MapData> arrayMap(ValueReader<?> keyReader, ValueReader<?> valueReader) {
+ return new ArrayMapReader(keyReader, valueReader);
+ }
+
+ static ValueReader<MapData> map(ValueReader<?> keyReader, ValueReader<?> valueReader) {
+ return new MapReader(keyReader, valueReader);
+ }
+
+ static ValueReader<RowData> struct(
+ List<ValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
+ return new StructReader(readers, struct, idToConstant);
+ }
+
+ private static class StringReader implements ValueReader<StringData> {
+ private static final StringReader INSTANCE = new StringReader();
+
+ private StringReader() {}
+
+ @Override
+ public StringData read(Decoder decoder, Object reuse) throws IOException {
+ // use the decoder's readString(Utf8) method because it may be a resolving decoder
+ Utf8 utf8 = null;
+ if (reuse instanceof StringData) {
+ utf8 = new Utf8(((StringData) reuse).toBytes());
+ }
+
+ Utf8 string = decoder.readString(utf8);
+ return StringData.fromBytes(string.getBytes(), 0, string.getByteLength());
+ }
+ }
+
+ private static class EnumReader implements ValueReader<StringData> {
+ private final StringData[] symbols;
+
+ private EnumReader(List<String> symbols) {
+ this.symbols = new StringData[symbols.size()];
+ for (int i = 0; i < this.symbols.length; i += 1) {
+ this.symbols[i] = StringData.fromBytes(symbols.get(i).getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ @Override
+ public StringData read(Decoder decoder, Object ignore) throws IOException {
+ int index = decoder.readEnum();
+ return symbols[index];
+ }
+ }
+
+ private static class DecimalReader implements ValueReader<DecimalData> {
+ private final ValueReader<byte[]> bytesReader;
+ private final int precision;
+ private final int scale;
+
+ private DecimalReader(ValueReader<byte[]> bytesReader, int precision, int scale) {
+ this.bytesReader = bytesReader;
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData read(Decoder decoder, Object reuse) throws IOException {
+ byte[] bytes = bytesReader.read(decoder, null);
+ return DecimalData.fromBigDecimal(
+ new BigDecimal(new BigInteger(bytes), scale), precision, scale);
+ }
+ }
+
+ private static class TimeMicrosReader implements ValueReader<Integer> {
+ private static final TimeMicrosReader INSTANCE = new TimeMicrosReader();
+
+ @Override
+ public Integer read(Decoder decoder, Object reuse) throws IOException {
+ long micros = decoder.readLong();
+ // Flink only support time mills, just erase micros.
+ return (int) (micros / 1000);
+ }
+ }
+
+ private static class TimestampMillsReader implements ValueReader<TimestampData> {
+ private static final TimestampMillsReader INSTANCE = new TimestampMillsReader();
+
+ @Override
+ public TimestampData read(Decoder decoder, Object reuse) throws IOException {
+ return TimestampData.fromEpochMillis(decoder.readLong());
+ }
+ }
+
+ private static class TimestampMicrosReader implements ValueReader<TimestampData> {
+ private static final TimestampMicrosReader INSTANCE = new TimestampMicrosReader();
+
+ @Override
+ public TimestampData read(Decoder decoder, Object reuse) throws IOException {
+ long micros = decoder.readLong();
+ long mills = micros / 1000;
+ int nanos = ((int) (micros % 1000)) * 1000;
+ if (nanos < 0) {
+ nanos += 1_000_000;
+ mills -= 1;
+ }
+ return TimestampData.fromEpochMillis(mills, nanos);
+ }
+ }
+
+ private static class ArrayReader implements ValueReader<ArrayData> {
+ private final ValueReader<?> elementReader;
+ private final List<Object> reusedList = Lists.newArrayList();
+
+ private ArrayReader(ValueReader<?> elementReader) {
+ this.elementReader = elementReader;
+ }
+
+ @Override
+ public GenericArrayData read(Decoder decoder, Object reuse) throws IOException {
+ reusedList.clear();
+ long chunkLength = decoder.readArrayStart();
+
+ while (chunkLength > 0) {
+ for (int i = 0; i < chunkLength; i += 1) {
+ reusedList.add(elementReader.read(decoder, null));
+ }
+
+ chunkLength = decoder.arrayNext();
+ }
+
+ // this will convert the list to an array so it is okay to reuse the list
+ return new GenericArrayData(reusedList.toArray());
+ }
+ }
+
+ private static MapData kvArrayToMap(List<Object> keyList, List<Object> valueList) {
+ Map<Object, Object> map = Maps.newHashMap();
+ Object[] keys = keyList.toArray();
+ Object[] values = valueList.toArray();
+ for (int i = 0; i < keys.length; i++) {
+ map.put(keys[i], values[i]);
+ }
+
+ return new GenericMapData(map);
+ }
+
+ private static class ArrayMapReader implements ValueReader<MapData> {
+ private final ValueReader<?> keyReader;
+ private final ValueReader<?> valueReader;
+
+ private final List<Object> reusedKeyList = Lists.newArrayList();
+ private final List<Object> reusedValueList = Lists.newArrayList();
+
+ private ArrayMapReader(ValueReader<?> keyReader, ValueReader<?> valueReader) {
+ this.keyReader = keyReader;
+ this.valueReader = valueReader;
+ }
+
+ @Override
+ public MapData read(Decoder decoder, Object reuse) throws IOException {
+ reusedKeyList.clear();
+ reusedValueList.clear();
+
+ long chunkLength = decoder.readArrayStart();
+
+ while (chunkLength > 0) {
+ for (int i = 0; i < chunkLength; i += 1) {
+ reusedKeyList.add(keyReader.read(decoder, null));
+ reusedValueList.add(valueReader.read(decoder, null));
+ }
+
+ chunkLength = decoder.arrayNext();
+ }
+
+ return kvArrayToMap(reusedKeyList, reusedValueList);
+ }
+ }
+
+ private static class MapReader implements ValueReader<MapData> {
+ private final ValueReader<?> keyReader;
+ private final ValueReader<?> valueReader;
+
+ private final List<Object> reusedKeyList = Lists.newArrayList();
+ private final List<Object> reusedValueList = Lists.newArrayList();
+
+ private MapReader(ValueReader<?> keyReader, ValueReader<?> valueReader) {
+ this.keyReader = keyReader;
+ this.valueReader = valueReader;
+ }
+
+ @Override
+ public MapData read(Decoder decoder, Object reuse) throws IOException {
+ reusedKeyList.clear();
+ reusedValueList.clear();
+
+ long chunkLength = decoder.readMapStart();
+
+ while (chunkLength > 0) {
+ for (int i = 0; i < chunkLength; i += 1) {
+ reusedKeyList.add(keyReader.read(decoder, null));
+ reusedValueList.add(valueReader.read(decoder, null));
+ }
+
+ chunkLength = decoder.mapNext();
+ }
+
+ return kvArrayToMap(reusedKeyList, reusedValueList);
+ }
+ }
+
+ private static class StructReader extends ValueReaders.StructReader<RowData> {
+ private final int numFields;
+
+ private StructReader(
+ List<ValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
+ super(readers, struct, idToConstant);
+ this.numFields = readers.size();
+ }
+
+ @Override
+ protected RowData reuseOrCreate(Object reuse) {
+ if (reuse instanceof GenericRowData && ((GenericRowData) reuse).getArity() == numFields) {
+ return (GenericRowData) reuse;
+ }
+ return new GenericRowData(numFields);
+ }
+
+ @Override
+ protected Object get(RowData struct, int pos) {
+ return null;
+ }
+
+ @Override
+ protected void set(RowData struct, int pos, Object value) {
+ ((GenericRowData) struct).setField(pos, value);
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueWriters.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueWriters.java
new file mode 100644
index 0000000000..4e86ecce28
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueWriters.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.util.List;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.util.Utf8;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.iceberg.avro.ValueWriter;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.util.DecimalUtil;
+
+public class FlinkValueWriters {
+
+ private FlinkValueWriters() {}
+
+ static ValueWriter<StringData> strings() {
+ return StringWriter.INSTANCE;
+ }
+
+ static ValueWriter<Integer> timeMicros() {
+ return TimeMicrosWriter.INSTANCE;
+ }
+
+ static ValueWriter<TimestampData> timestampMicros() {
+ return TimestampMicrosWriter.INSTANCE;
+ }
+
+ static ValueWriter<DecimalData> decimal(int precision, int scale) {
+ return new DecimalWriter(precision, scale);
+ }
+
+ static <T> ValueWriter<ArrayData> array(ValueWriter<T> elementWriter, LogicalType elementType) {
+ return new ArrayWriter<>(elementWriter, elementType);
+ }
+
+ static <K, V> ValueWriter<MapData> arrayMap(
+ ValueWriter<K> keyWriter,
+ LogicalType keyType,
+ ValueWriter<V> valueWriter,
+ LogicalType valueType) {
+ return new ArrayMapWriter<>(keyWriter, keyType, valueWriter, valueType);
+ }
+
+ static <K, V> ValueWriter<MapData> map(
+ ValueWriter<K> keyWriter,
+ LogicalType keyType,
+ ValueWriter<V> valueWriter,
+ LogicalType valueType) {
+ return new MapWriter<>(keyWriter, keyType, valueWriter, valueType);
+ }
+
+ static ValueWriter<RowData> row(List<ValueWriter<?>> writers, List<LogicalType> types) {
+ return new RowWriter(writers, types);
+ }
+
+ private static class StringWriter implements ValueWriter<StringData> {
+ private static final StringWriter INSTANCE = new StringWriter();
+
+ private StringWriter() {}
+
+ @Override
+ public void write(StringData s, Encoder encoder) throws IOException {
+ // toBytes is cheaper than Avro calling toString, which incurs encoding costs
+ encoder.writeString(new Utf8(s.toBytes()));
+ }
+ }
+
+ private static class DecimalWriter implements ValueWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+ private final ThreadLocal<byte[]> bytes;
+
+ private DecimalWriter(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ this.bytes =
+ ThreadLocal.withInitial(() -> new byte[TypeUtil.decimalRequiredBytes(precision)]);
+ }
+
+ @Override
+ public void write(DecimalData d, Encoder encoder) throws IOException {
+ encoder.writeFixed(
+ DecimalUtil.toReusedFixLengthBytes(precision, scale, d.toBigDecimal(), bytes.get()));
+ }
+ }
+
+ private static class TimeMicrosWriter implements ValueWriter<Integer> {
+ private static final TimeMicrosWriter INSTANCE = new TimeMicrosWriter();
+
+ @Override
+ public void write(Integer timeMills, Encoder encoder) throws IOException {
+ encoder.writeLong(timeMills * 1000L);
+ }
+ }
+
+ private static class TimestampMicrosWriter implements ValueWriter<TimestampData> {
+ private static final TimestampMicrosWriter INSTANCE = new TimestampMicrosWriter();
+
+ @Override
+ public void write(TimestampData timestampData, Encoder encoder) throws IOException {
+ long micros =
+ timestampData.getMillisecond() * 1000 + timestampData.getNanoOfMillisecond() / 1000;
+ encoder.writeLong(micros);
+ }
+ }
+
+ private static class ArrayWriter<T> implements ValueWriter<ArrayData> {
+ private final ValueWriter<T> elementWriter;
+ private final ArrayData.ElementGetter elementGetter;
+
+ private ArrayWriter(ValueWriter<T> elementWriter, LogicalType elementType) {
+ this.elementWriter = elementWriter;
+ this.elementGetter = ArrayData.createElementGetter(elementType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void write(ArrayData array, Encoder encoder) throws IOException {
+ encoder.writeArrayStart();
+ int numElements = array.size();
+ encoder.setItemCount(numElements);
+ for (int i = 0; i < numElements; i += 1) {
+ encoder.startItem();
+ elementWriter.write((T) elementGetter.getElementOrNull(array, i), encoder);
+ }
+ encoder.writeArrayEnd();
+ }
+ }
+
+ private static class ArrayMapWriter<K, V> implements ValueWriter<MapData> {
+ private final ValueWriter<K> keyWriter;
+ private final ValueWriter<V> valueWriter;
+ private final ArrayData.ElementGetter keyGetter;
+ private final ArrayData.ElementGetter valueGetter;
+
+ private ArrayMapWriter(
+ ValueWriter<K> keyWriter,
+ LogicalType keyType,
+ ValueWriter<V> valueWriter,
+ LogicalType valueType) {
+ this.keyWriter = keyWriter;
+ this.keyGetter = ArrayData.createElementGetter(keyType);
+ this.valueWriter = valueWriter;
+ this.valueGetter = ArrayData.createElementGetter(valueType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void write(MapData map, Encoder encoder) throws IOException {
+ encoder.writeArrayStart();
+ int numElements = map.size();
+ encoder.setItemCount(numElements);
+ ArrayData keyArray = map.keyArray();
+ ArrayData valueArray = map.valueArray();
+ for (int i = 0; i < numElements; i += 1) {
+ encoder.startItem();
+ keyWriter.write((K) keyGetter.getElementOrNull(keyArray, i), encoder);
+ valueWriter.write((V) valueGetter.getElementOrNull(valueArray, i), encoder);
+ }
+ encoder.writeArrayEnd();
+ }
+ }
+
+ private static class MapWriter<K, V> implements ValueWriter<MapData> {
+ private final ValueWriter<K> keyWriter;
+ private final ValueWriter<V> valueWriter;
+ private final ArrayData.ElementGetter keyGetter;
+ private final ArrayData.ElementGetter valueGetter;
+
+ private MapWriter(
+ ValueWriter<K> keyWriter,
+ LogicalType keyType,
+ ValueWriter<V> valueWriter,
+ LogicalType valueType) {
+ this.keyWriter = keyWriter;
+ this.keyGetter = ArrayData.createElementGetter(keyType);
+ this.valueWriter = valueWriter;
+ this.valueGetter = ArrayData.createElementGetter(valueType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void write(MapData map, Encoder encoder) throws IOException {
+ encoder.writeMapStart();
+ int numElements = map.size();
+ encoder.setItemCount(numElements);
+ ArrayData keyArray = map.keyArray();
+ ArrayData valueArray = map.valueArray();
+ for (int i = 0; i < numElements; i += 1) {
+ encoder.startItem();
+ keyWriter.write((K) keyGetter.getElementOrNull(keyArray, i), encoder);
+ valueWriter.write((V) valueGetter.getElementOrNull(valueArray, i), encoder);
+ }
+ encoder.writeMapEnd();
+ }
+ }
+
+ static class RowWriter implements ValueWriter<RowData> {
+ private final ValueWriter<?>[] writers;
+ private final RowData.FieldGetter[] getters;
+
+ private RowWriter(List<ValueWriter<?>> writers, List<LogicalType> types) {
+ this.writers = (ValueWriter<?>[]) Array.newInstance(ValueWriter.class, writers.size());
+ this.getters = new RowData.FieldGetter[writers.size()];
+ for (int i = 0; i < writers.size(); i += 1) {
+ this.writers[i] = writers.get(i);
+ this.getters[i] = RowData.createFieldGetter(types.get(i), i);
+ }
+ }
+
+ @Override
+ public void write(RowData row, Encoder encoder) throws IOException {
+ for (int i = 0; i < writers.length; i += 1) {
+ if (row.isNullAt(i)) {
+ writers[i].write(null, encoder);
+ } else {
+ write(row, i, writers[i], encoder);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> void write(RowData row, int pos, ValueWriter<T> writer, Encoder encoder)
+ throws IOException {
+ writer.write((T) getters[pos].getFieldOrNull(row), encoder);
+ }
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java
new file mode 100644
index 0000000000..33feb2e321
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.Deque;
+import java.util.List;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.RowType.RowField;
+import org.apache.iceberg.avro.AvroSchemaUtil;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+public class ParquetWithFlinkSchemaVisitor<T> {
+ private final Deque<String> fieldNames = Lists.newLinkedList();
+
+ public static <T> T visit(
+ LogicalType sType, Type type, ParquetWithFlinkSchemaVisitor<T> visitor) {
+ Preconditions.checkArgument(sType != null, "Invalid DataType: null");
+ if (type instanceof MessageType) {
+ Preconditions.checkArgument(
+ sType instanceof RowType, "Invalid struct: %s is not a struct", sType);
+ RowType struct = (RowType) sType;
+ return visitor.message(
+ struct, (MessageType) type, visitFields(struct, type.asGroupType(), visitor));
+ } else if (type.isPrimitive()) {
+ return visitor.primitive(sType, type.asPrimitiveType());
+ } else {
+ // if not a primitive, the typeId must be a group
+ GroupType group = type.asGroupType();
+ OriginalType annotation = group.getOriginalType();
+ if (annotation != null) {
+ switch (annotation) {
+ case LIST:
+ Preconditions.checkArgument(
+ !group.isRepetition(Type.Repetition.REPEATED),
+ "Invalid list: top-level group is repeated: %s",
+ group);
+ Preconditions.checkArgument(
+ group.getFieldCount() == 1,
+ "Invalid list: does not contain single repeated field: %s",
+ group);
+
+ GroupType repeatedElement = group.getFields().get(0).asGroupType();
+ Preconditions.checkArgument(
+ repeatedElement.isRepetition(Type.Repetition.REPEATED),
+ "Invalid list: inner group is not repeated");
+ Preconditions.checkArgument(
+ repeatedElement.getFieldCount() <= 1,
+ "Invalid list: repeated group is not a single field: %s",
+ group);
+
+ Preconditions.checkArgument(
+ sType instanceof ArrayType, "Invalid list: %s is not an array", sType);
+ ArrayType array = (ArrayType) sType;
+ RowType.RowField element =
+ new RowField(
+ "element", array.getElementType(), "element of " + array.asSummaryString());
+
+ visitor.fieldNames.push(repeatedElement.getName());
+ try {
+ T elementResult = null;
+ if (repeatedElement.getFieldCount() > 0) {
+ elementResult = visitField(element, repeatedElement.getType(0), visitor);
+ }
+
+ return visitor.list(array, group, elementResult);
+
+ } finally {
+ visitor.fieldNames.pop();
+ }
+
+ case MAP:
+ Preconditions.checkArgument(
+ !group.isRepetition(Type.Repetition.REPEATED),
+ "Invalid map: top-level group is repeated: %s",
+ group);
+ Preconditions.checkArgument(
+ group.getFieldCount() == 1,
+ "Invalid map: does not contain single repeated field: %s",
+ group);
+
+ GroupType repeatedKeyValue = group.getType(0).asGroupType();
+ Preconditions.checkArgument(
+ repeatedKeyValue.isRepetition(Type.Repetition.REPEATED),
+ "Invalid map: inner group is not repeated");
+ Preconditions.checkArgument(
+ repeatedKeyValue.getFieldCount() <= 2,
+ "Invalid map: repeated group does not have 2 fields");
+
+ Preconditions.checkArgument(
+ sType instanceof MapType, "Invalid map: %s is not a map", sType);
+ MapType map = (MapType) sType;
+ RowField keyField =
+ new RowField("key", map.getKeyType(), "key of " + map.asSummaryString());
+ RowField valueField =
+ new RowField("value", map.getValueType(), "value of " + map.asSummaryString());
+
+ visitor.fieldNames.push(repeatedKeyValue.getName());
+ try {
+ T keyResult = null;
+ T valueResult = null;
+ switch (repeatedKeyValue.getFieldCount()) {
+ case 2:
+ // if there are 2 fields, both key and value are projected
+ keyResult = visitField(keyField, repeatedKeyValue.getType(0), visitor);
+ valueResult = visitField(valueField, repeatedKeyValue.getType(1), visitor);
+ break;
+ case 1:
+ // if there is just one, use the name to determine what it is
+ Type keyOrValue = repeatedKeyValue.getType(0);
+ if (keyOrValue.getName().equalsIgnoreCase("key")) {
+ keyResult = visitField(keyField, keyOrValue, visitor);
+ // value result remains null
+ } else {
+ valueResult = visitField(valueField, keyOrValue, visitor);
+ // key result remains null
+ }
+ break;
+ default:
+ // both results will remain null
+ }
+
+ return visitor.map(map, group, keyResult, valueResult);
+
+ } finally {
+ visitor.fieldNames.pop();
+ }
+
+ default:
+ }
+ }
+ Preconditions.checkArgument(
+ sType instanceof RowType, "Invalid struct: %s is not a struct", sType);
+ RowType struct = (RowType) sType;
+ return visitor.struct(struct, group, visitFields(struct, group, visitor));
+ }
+ }
+
+ private static <T> T visitField(
+ RowType.RowField sField, Type field, ParquetWithFlinkSchemaVisitor<T> visitor) {
+ visitor.fieldNames.push(field.getName());
+ try {
+ return visit(sField.getType(), field, visitor);
+ } finally {
+ visitor.fieldNames.pop();
+ }
+ }
+
+ private static <T> List<T> visitFields(
+ RowType struct, GroupType group, ParquetWithFlinkSchemaVisitor<T> visitor) {
+ List<RowType.RowField> sFields = struct.getFields();
+ Preconditions.checkArgument(
+ sFields.size() == group.getFieldCount(), "Structs do not match: %s and %s", struct, group);
+ List<T> results = Lists.newArrayListWithExpectedSize(group.getFieldCount());
+ for (int i = 0; i < sFields.size(); i += 1) {
+ Type field = group.getFields().get(i);
+ RowType.RowField sField = sFields.get(i);
+ Preconditions.checkArgument(
+ field.getName().equals(AvroSchemaUtil.makeCompatibleName(sField.getName())),
+ "Structs do not match: field %s != %s",
+ field.getName(),
+ sField.getName());
+ results.add(visitField(sField, field, visitor));
+ }
+
+ return results;
+ }
+
+ public T message(RowType sStruct, MessageType message, List<T> fields) {
+ return null;
+ }
+
+ public T struct(RowType sStruct, GroupType struct, List<T> fields) {
+ return null;
+ }
+
+ public T list(ArrayType sArray, GroupType array, T element) {
+ return null;
+ }
+
+ public T map(MapType sMap, GroupType map, T key, T value) {
+ return null;
+ }
+
+ public T primitive(LogicalType sPrimitive, PrimitiveType primitive) {
+ return null;
+ }
+
+ protected String[] currentPath() {
+ return Lists.newArrayList(fieldNames.descendingIterator()).toArray(new String[0]);
+ }
+
+ protected String[] path(String name) {
+ List<String> list = Lists.newArrayList(fieldNames.descendingIterator());
+ list.add(name);
+ return list.toArray(new String[0]);
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/RowDataProjection.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/RowDataProjection.java
new file mode 100644
index 0000000000..33816c97ac
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/RowDataProjection.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RawValueData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.StringUtils;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.flink.FlinkSchemaUtil;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+
+public class RowDataProjection implements RowData {
+ /**
+ * Creates a projecting wrapper for {@link RowData} rows.
+ *
+ * <p>This projection will not project the nested children types of repeated types like lists and
+ * maps.
+ *
+ * @param schema schema of rows wrapped by this projection
+ * @param projectedSchema result schema of the projected rows
+ * @return a wrapper to project rows
+ */
+ public static RowDataProjection create(Schema schema, Schema projectedSchema) {
+ return RowDataProjection.create(
+ FlinkSchemaUtil.convert(schema), schema.asStruct(), projectedSchema.asStruct());
+ }
+
+ /**
+ * Creates a projecting wrapper for {@link RowData} rows.
+ *
+ * <p>This projection will not project the nested children types of repeated types like lists and
+ * maps.
+ *
+ * @param rowType flink row type of rows wrapped by this projection
+ * @param schema schema of rows wrapped by this projection
+ * @param projectedSchema result schema of the projected rows
+ * @return a wrapper to project rows
+ */
+ public static RowDataProjection create(
+ RowType rowType, Types.StructType schema, Types.StructType projectedSchema) {
+ return new RowDataProjection(rowType, schema, projectedSchema);
+ }
+
+ private final RowData.FieldGetter[] getters;
+ private RowData rowData;
+
+ private RowDataProjection(
+ RowType rowType, Types.StructType rowStruct, Types.StructType projectType) {
+ Map<Integer, Integer> fieldIdToPosition = Maps.newHashMap();
+ for (int i = 0; i < rowStruct.fields().size(); i++) {
+ fieldIdToPosition.put(rowStruct.fields().get(i).fieldId(), i);
+ }
+
+ this.getters = new RowData.FieldGetter[projectType.fields().size()];
+ for (int i = 0; i < getters.length; i++) {
+ Types.NestedField projectField = projectType.fields().get(i);
+ Types.NestedField rowField = rowStruct.field(projectField.fieldId());
+
+ Preconditions.checkNotNull(
+ rowField,
+ "Cannot locate the project field <%s> in the iceberg struct <%s>",
+ projectField,
+ rowStruct);
+
+ getters[i] =
+ createFieldGetter(
+ rowType, fieldIdToPosition.get(projectField.fieldId()), rowField, projectField);
+ }
+ }
+
+ private static RowData.FieldGetter createFieldGetter(
+ RowType rowType, int position, Types.NestedField rowField, Types.NestedField projectField) {
+ Preconditions.checkArgument(
+ rowField.type().typeId() == projectField.type().typeId(),
+ "Different iceberg type between row field <%s> and project field <%s>",
+ rowField,
+ projectField);
+
+ switch (projectField.type().typeId()) {
+ case STRUCT:
+ RowType nestedRowType = (RowType) rowType.getTypeAt(position);
+ return row -> {
+ // null nested struct value
+ if (row.isNullAt(position)) {
+ return null;
+ }
+
+ RowData nestedRow = row.getRow(position, nestedRowType.getFieldCount());
+ return RowDataProjection.create(
+ nestedRowType, rowField.type().asStructType(), projectField.type().asStructType())
+ .wrap(nestedRow);
+ };
+
+ case MAP:
+ Types.MapType projectedMap = projectField.type().asMapType();
+ Types.MapType originalMap = rowField.type().asMapType();
+
+ boolean keyProjectable =
+ !projectedMap.keyType().isNestedType()
+ || projectedMap.keyType().equals(originalMap.keyType());
+ boolean valueProjectable =
+ !projectedMap.valueType().isNestedType()
+ || projectedMap.valueType().equals(originalMap.valueType());
+ Preconditions.checkArgument(
+ keyProjectable && valueProjectable,
+ "Cannot project a partial map key or value with non-primitive type. Trying to project <%s> out of <%s>",
+ projectField,
+ rowField);
+
+ return RowData.createFieldGetter(rowType.getTypeAt(position), position);
+
+ case LIST:
+ Types.ListType projectedList = projectField.type().asListType();
+ Types.ListType originalList = rowField.type().asListType();
+
+ boolean elementProjectable =
+ !projectedList.elementType().isNestedType()
+ || projectedList.elementType().equals(originalList.elementType());
+ Preconditions.checkArgument(
+ elementProjectable,
+ "Cannot project a partial list element with non-primitive type. Trying to project <%s> out of <%s>",
+ projectField,
+ rowField);
+
+ return RowData.createFieldGetter(rowType.getTypeAt(position), position);
+
+ default:
+ return RowData.createFieldGetter(rowType.getTypeAt(position), position);
+ }
+ }
+
+ public RowData wrap(RowData row) {
+ // StructProjection allow wrapping null root struct object.
+ // See more discussions in https://github.com/apache/iceberg/pull/7517.
+ // RowDataProjection never allowed null root object to be wrapped.
+ // Hence, it is fine to enforce strict Preconditions check here.
+ Preconditions.checkArgument(row != null, "Invalid row data: null");
+ this.rowData = row;
+ return this;
+ }
+
+ private Object getValue(int pos) {
+ Preconditions.checkState(rowData != null, "Row data not wrapped");
+ return getters[pos].getFieldOrNull(rowData);
+ }
+
+ @Override
+ public int getArity() {
+ return getters.length;
+ }
+
+ @Override
+ public RowKind getRowKind() {
+ Preconditions.checkState(rowData != null, "Row data not wrapped");
+ return rowData.getRowKind();
+ }
+
+ @Override
+ public void setRowKind(RowKind kind) {
+ throw new UnsupportedOperationException("Cannot set row kind in the RowDataProjection");
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ return getValue(pos) == null;
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ return (boolean) getValue(pos);
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ return (byte) getValue(pos);
+ }
+
+ @Override
+ public short getShort(int pos) {
+ return (short) getValue(pos);
+ }
+
+ @Override
+ public int getInt(int pos) {
+ return (int) getValue(pos);
+ }
+
+ @Override
+ public long getLong(int pos) {
+ return (long) getValue(pos);
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ return (float) getValue(pos);
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ return (double) getValue(pos);
+ }
+
+ @Override
+ public StringData getString(int pos) {
+ return (StringData) getValue(pos);
+ }
+
+ @Override
+ public DecimalData getDecimal(int pos, int precision, int scale) {
+ return (DecimalData) getValue(pos);
+ }
+
+ @Override
+ public TimestampData getTimestamp(int pos, int precision) {
+ return (TimestampData) getValue(pos);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> RawValueData<T> getRawValue(int pos) {
+ return (RawValueData<T>) getValue(pos);
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ return (byte[]) getValue(pos);
+ }
+
+ @Override
+ public ArrayData getArray(int pos) {
+ return (ArrayData) getValue(pos);
+ }
+
+ @Override
+ public MapData getMap(int pos) {
+ return (MapData) getValue(pos);
+ }
+
+ @Override
+ public RowData getRow(int pos, int numFields) {
+ return (RowData) getValue(pos);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (!(o instanceof RowDataProjection)) {
+ return false;
+ }
+
+ RowDataProjection that = (RowDataProjection) o;
+ return deepEquals(that);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hashCode(getRowKind());
+ for (int pos = 0; pos < getArity(); pos++) {
+ if (!isNullAt(pos)) {
+ // Arrays.deepHashCode handles array object properly
+ result = 31 * result + Arrays.deepHashCode(new Object[] {getValue(pos)});
+ }
+ }
+
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getRowKind().shortString()).append("(");
+ for (int pos = 0; pos < getArity(); pos++) {
+ if (pos != 0) {
+ sb.append(",");
+ }
+ // copied the behavior from Flink GenericRowData
+ sb.append(StringUtils.arrayAwareToString(getValue(pos)));
+ }
+
+ sb.append(")");
+ return sb.toString();
+ }
+
+ private boolean deepEquals(RowDataProjection other) {
+ if (getRowKind() != other.getRowKind()) {
+ return false;
+ }
+
+ if (getArity() != other.getArity()) {
+ return false;
+ }
+
+ for (int pos = 0; pos < getArity(); ++pos) {
+ if (isNullAt(pos) && other.isNullAt(pos)) {
+ continue;
+ }
+
+ if ((isNullAt(pos) && !other.isNullAt(pos)) || (!isNullAt(pos) && other.isNullAt(pos))) {
+ return false;
+ }
+
+ // Objects.deepEquals handles array object properly
+ if (!Objects.deepEquals(getValue(pos), other.getValue(pos))) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/RowDataUtil.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/RowDataUtil.java
new file mode 100644
index 0000000000..c5cb51b7ea
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/RowDataUtil.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.util.Utf8;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.DateTimeUtil;
+
+public class RowDataUtil {
+
+ private RowDataUtil() {}
+
+ public static Object convertConstant(Type type, Object value) {
+ if (value == null) {
+ return null;
+ }
+
+ switch (type.typeId()) {
+ case DECIMAL: // DecimalData
+ Types.DecimalType decimal = (Types.DecimalType) type;
+ return DecimalData.fromBigDecimal((BigDecimal) value, decimal.precision(), decimal.scale());
+ case STRING: // StringData
+ if (value instanceof Utf8) {
+ Utf8 utf8 = (Utf8) value;
+ return StringData.fromBytes(utf8.getBytes(), 0, utf8.getByteLength());
+ }
+ return StringData.fromString(value.toString());
+ case FIXED: // byte[]
+ if (value instanceof byte[]) {
+ return value;
+ } else if (value instanceof GenericData.Fixed) {
+ return ((GenericData.Fixed) value).bytes();
+ }
+ return ByteBuffers.toByteArray((ByteBuffer) value);
+ case BINARY: // byte[]
+ return ByteBuffers.toByteArray((ByteBuffer) value);
+ case TIME: // int mills instead of long
+ return (int) ((Long) value / 1000);
+ case TIMESTAMP: // TimestampData
+ return TimestampData.fromLocalDateTime(DateTimeUtil.timestampFromMicros((Long) value));
+ default:
+ }
+ return value;
+ }
+
+ /**
+ * Similar to the private {@link RowDataSerializer#copyRowData(RowData, RowData)} method. This
+ * skips the check the arity of rowType and from, because the from RowData may contains additional
+ * column for position deletes. Using {@link RowDataSerializer#copy(RowData, RowData)} will fail
+ * the arity check.
+ */
+ public static RowData clone(
+ RowData from, RowData reuse, RowType rowType, TypeSerializer[] fieldSerializers) {
+ GenericRowData ret;
+ if (reuse instanceof GenericRowData) {
+ ret = (GenericRowData) reuse;
+ } else {
+ ret = new GenericRowData(from.getArity());
+ }
+ ret.setRowKind(from.getRowKind());
+ for (int i = 0; i < rowType.getFieldCount(); i++) {
+ if (!from.isNullAt(i)) {
+ RowData.FieldGetter getter = RowData.createFieldGetter(rowType.getTypeAt(i), i);
+ ret.setField(i, fieldSerializers[i].copy(getter.getFieldOrNull(from)));
+ } else {
+ ret.setField(i, null);
+ }
+ }
+ return ret;
+ }
+}
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/StructRowData.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/StructRowData.java
new file mode 100644
index 0000000000..1019285018
--- /dev/null
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/data/StructRowData.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericArrayData;
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RawValueData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.types.RowKind;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+
+@Internal
+public class StructRowData implements RowData {
+ private final Types.StructType type;
+ private RowKind kind;
+ private StructLike struct;
+
+ public StructRowData(Types.StructType type) {
+ this(type, RowKind.INSERT);
+ }
+
+ public StructRowData(Types.StructType type, RowKind kind) {
+ this(type, null, kind);
+ }
+
+ private StructRowData(Types.StructType type, StructLike struct) {
+ this(type, struct, RowKind.INSERT);
+ }
+
+ private StructRowData(Types.StructType type, StructLike struct, RowKind kind) {
+ this.type = type;
+ this.struct = struct;
+ this.kind = kind;
+ }
+
+ public StructRowData setStruct(StructLike newStruct) {
+ this.struct = newStruct;
+ return this;
+ }
+
+ @Override
+ public int getArity() {
+ return struct.size();
+ }
+
+ @Override
+ public RowKind getRowKind() {
+ return kind;
+ }
+
+ @Override
+ public void setRowKind(RowKind newKind) {
+ Preconditions.checkNotNull(newKind, "kind can not be null");
+ this.kind = newKind;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ return struct.get(pos, Object.class) == null;
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ return struct.get(pos, Boolean.class);
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ return (byte) (int) struct.get(pos, Integer.class);
+ }
+
+ @Override
+ public short getShort(int pos) {
+ return (short) (int) struct.get(pos, Integer.class);
+ }
+
+ @Override
+ public int getInt(int pos) {
+ Object integer = struct.get(pos, Object.class);
+
+ if (integer instanceof Integer) {
+ return (int) integer;
+ } else if (integer instanceof LocalDate) {
+ return (int) ((LocalDate) integer).toEpochDay();
+ } else if (integer instanceof LocalTime) {
+ return (int) (((LocalTime) integer).toNanoOfDay() / 1000_000);
+ } else {
+ throw new IllegalStateException(
+ "Unknown type for int field. Type name: " + integer.getClass().getName());
+ }
+ }
+
+ @Override
+ public long getLong(int pos) {
+ Object longVal = struct.get(pos, Object.class);
+
+ if (longVal instanceof Long) {
+ return (long) longVal;
+ } else if (longVal instanceof OffsetDateTime) {
+ return Duration.between(Instant.EPOCH, (OffsetDateTime) longVal).toNanos() / 1000;
+ } else if (longVal instanceof LocalDate) {
+ return ((LocalDate) longVal).toEpochDay();
+ } else if (longVal instanceof LocalTime) {
+ return ((LocalTime) longVal).toNanoOfDay();
+ } else if (longVal instanceof LocalDateTime) {
+ return Duration.between(Instant.EPOCH, ((LocalDateTime) longVal).atOffset(ZoneOffset.UTC))
+ .toNanos()
+ / 1000;
+ } else {
+ throw new IllegalStateException(
+ "Unknown type for long field. Type name: " + longVal.getClass().getName());
+ }
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ return struct.get(pos, Float.class);
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ return struct.get(pos, Double.class);
+ }
+
+ @Override
+ public StringData getString(int pos) {
+ return isNullAt(pos) ? null : getStringDataInternal(pos);
+ }
+
+ private StringData getStringDataInternal(int pos) {
+ CharSequence seq = struct.get(pos, CharSequence.class);
+ return StringData.fromString(seq.toString());
+ }
+
+ @Override
+ public DecimalData getDecimal(int pos, int precision, int scale) {
+ return isNullAt(pos)
+ ? null
+ : DecimalData.fromBigDecimal(getDecimalInternal(pos), precision, scale);
+ }
+
+ private BigDecimal getDecimalInternal(int pos) {
+ return struct.get(pos, BigDecimal.class);
+ }
+
+ @Override
+ public TimestampData getTimestamp(int pos, int precision) {
+ long timeLong = getLong(pos);
+ return TimestampData.fromEpochMillis(timeLong / 1000, (int) (timeLong % 1000) * 1000);
+ }
+
+ @Override
+ public <T> RawValueData<T> getRawValue(int pos) {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ return isNullAt(pos) ? null : getBinaryInternal(pos);
+ }
+
+ private byte[] getBinaryInternal(int pos) {
+ Object bytes = struct.get(pos, Object.class);
+
+ // should only be either ByteBuffer or byte[]
+ if (bytes instanceof ByteBuffer) {
+ return ByteBuffers.toByteArray((ByteBuffer) bytes);
+ } else if (bytes instanceof byte[]) {
+ return (byte[]) bytes;
+ } else if (bytes instanceof UUID) {
+ UUID uuid = (UUID) bytes;
+ ByteBuffer bb = ByteBuffer.allocate(16);
+ bb.putLong(uuid.getMostSignificantBits());
+ bb.putLong(uuid.getLeastSignificantBits());
+ return bb.array();
+ } else {
+ throw new IllegalStateException(
+ "Unknown type for binary field. Type name: " + bytes.getClass().getName());
+ }
+ }
+
+ @Override
+ public ArrayData getArray(int pos) {
+ return isNullAt(pos)
+ ? null
+ : (ArrayData)
+ convertValue(type.fields().get(pos).type().asListType(), struct.get(pos, List.class));
+ }
+
+ @Override
+ public MapData getMap(int pos) {
+ return isNullAt(pos)
+ ? null
+ : (MapData)
+ convertValue(type.fields().get(pos).type().asMapType(), struct.get(pos, Map.class));
+ }
+
+ @Override
+ public RowData getRow(int pos, int numFields) {
+ return isNullAt(pos) ? null : getStructRowData(pos, numFields);
+ }
+
+ private StructRowData getStructRowData(int pos, int numFields) {
+ return new StructRowData(
+ type.fields().get(pos).type().asStructType(), struct.get(pos, StructLike.class));
+ }
+
+ private Object convertValue(Type elementType, Object value) {
+ switch (elementType.typeId()) {
+ case BOOLEAN:
+ case INTEGER:
+ case DATE:
+ case TIME:
+ case LONG:
+ case FLOAT:
+ case DOUBLE:
+ case DECIMAL:
+ return value;
+ case TIMESTAMP:
+ long millisecond = (long) value / 1000;
+ int nanoOfMillisecond = (int) ((Long) value % 1000) * 1000;
+ return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond);
+ case STRING:
+ return StringData.fromString(value.toString());
+ case FIXED:
+ case BINARY:
+ return ByteBuffers.toByteArray((ByteBuffer) value);
+ case STRUCT:
+ return new StructRowData(elementType.asStructType(), (StructLike) value);
+ case LIST:
+ List<Object> list = (List<Object>) value;
+ Object[] array = new Object[list.size()];
+
+ int index = 0;
+ for (Object element : list) {
+ if (element == null) {
+ array[index] = null;
+ } else {
+ array[index] = convertValue(elementType.asListType().elementType(), element);
+ }
+
+ index += 1;
+ }
+ return new GenericArrayData(array);
+ case MAP:
... 43119 lines suppressed ...