You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2026/08/28 12:17:13 UTC
(iceberg) 02/05: Flink: Add v2.2 directory
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 8dc651d81ab3b9337389a454bec21ffabfdf240c
Author: Talat Uyarer <ta...@apache.org>
AuthorDate: Thu Aug 27 09:33:01 2026 -0700
Flink: Add v2.2 directory
---
flink/v2.2/build.gradle | 266 +++
flink/v2.2/flink-runtime/LICENSE | 1047 +++++++++++
flink/v2.2/flink-runtime/NOTICE | 439 +++++
flink/v2.2/flink-runtime/runtime-deps.txt | 31 +
.../iceberg/flink/TestIcebergConnectorSmoke.java | 21 +
...namicRecordSerializerDeserializerBenchmark.java | 139 ++
.../sink/shuffle/MapRangePartitionerBenchmark.java | 121 ++
.../shuffle/SketchRangePartitionerBenchmark.java | 114 ++
.../StatisticsRecordSerializerBenchmark.java | 127 ++
.../runtime/types/FlinkScalaKryoInstantiator.java | 26 +
.../types/variant/BinaryVariantAccessorUtils.java | 86 +
.../org/apache/iceberg/flink/CatalogLoader.java | 215 +++
.../org/apache/iceberg/flink/FlinkCatalog.java | 879 +++++++++
.../apache/iceberg/flink/FlinkCatalogFactory.java | 215 +++
.../org/apache/iceberg/flink/FlinkConfParser.java | 297 +++
.../apache/iceberg/flink/FlinkConfigOptions.java | 113 ++
.../iceberg/flink/FlinkCreateTableOptions.java | 119 ++
.../iceberg/flink/FlinkDynamicTableFactory.java | 277 +++
.../iceberg/flink/FlinkEnvironmentContext.java | 31 +
.../org/apache/iceberg/flink/FlinkFilters.java | 266 +++
.../org/apache/iceberg/flink/FlinkFixupTypes.java | 50 +
.../org/apache/iceberg/flink/FlinkReadConf.java | 213 +++
.../org/apache/iceberg/flink/FlinkReadOptions.java | 118 ++
.../org/apache/iceberg/flink/FlinkRowData.java | 47 +
.../org/apache/iceberg/flink/FlinkSchemaUtil.java | 380 ++++
.../apache/iceberg/flink/FlinkSourceFilter.java | 49 +
.../org/apache/iceberg/flink/FlinkTypeToType.java | 215 +++
.../org/apache/iceberg/flink/FlinkTypeVisitor.java | 80 +
.../org/apache/iceberg/flink/FlinkWriteConf.java | 292 +++
.../apache/iceberg/flink/FlinkWriteOptions.java | 120 ++
.../org/apache/iceberg/flink/IcebergTableSink.java | 306 ++++
.../org/apache/iceberg/flink/RowDataWrapper.java | 162 ++
.../java/org/apache/iceberg/flink/TableLoader.java | 159 ++
.../org/apache/iceberg/flink/TypeToFlinkType.java | 152 ++
.../org/apache/iceberg/flink/actions/Actions.java | 52 +
.../flink/actions/RewriteDataFilesAction.java | 76 +
.../flink/data/AvroWithFlinkSchemaVisitor.java | 81 +
.../apache/iceberg/flink/data/FlinkAvroWriter.java | 179 ++
.../iceberg/flink/data/FlinkFormatModels.java | 64 +
.../apache/iceberg/flink/data/FlinkOrcReader.java | 138 ++
.../apache/iceberg/flink/data/FlinkOrcReaders.java | 290 +++
.../apache/iceberg/flink/data/FlinkOrcWriter.java | 172 ++
.../apache/iceberg/flink/data/FlinkOrcWriters.java | 368 ++++
.../iceberg/flink/data/FlinkParquetReaders.java | 901 +++++++++
.../iceberg/flink/data/FlinkParquetWriters.java | 687 +++++++
.../iceberg/flink/data/FlinkPlannedAvroReader.java | 202 +++
.../iceberg/flink/data/FlinkSchemaVisitor.java | 163 ++
.../iceberg/flink/data/FlinkValueReaders.java | 387 ++++
.../iceberg/flink/data/FlinkValueWriters.java | 314 ++++
.../flink/data/FlinkVariantShreddingAnalyzer.java | 72 +
.../flink/data/ParquetWithFlinkSchemaVisitor.java | 242 +++
.../iceberg/flink/data/RowDataProjection.java | 348 ++++
.../org/apache/iceberg/flink/data/RowDataUtil.java | 111 ++
.../apache/iceberg/flink/data/StructRowData.java | 346 ++++
.../iceberg/flink/data/VariantRowDataWrapper.java | 322 ++++
.../formats/avro/AvroToRowDataConverters.java | 303 ++++
.../iceberg/flink/formats/avro/JodaConverter.java | 69 +
.../formats/avro/RowDataToAvroConverters.java | 394 ++++
.../avro/typeutils/AvroSchemaConverter.java | 625 +++++++
.../maintenance/api/ConvertEqualityDeletes.java | 303 ++++
.../api/ConvertEqualityDeletesConfig.java | 90 +
.../flink/maintenance/api/DeleteOrphanFiles.java | 339 ++++
.../maintenance/api/DeleteOrphanFilesConfig.java | 216 +++
.../flink/maintenance/api/ExpireSnapshots.java | 164 ++
.../maintenance/api/ExpireSnapshotsConfig.java | 151 ++
.../maintenance/api/FlinkMaintenanceConfig.java | 138 ++
.../flink/maintenance/api/JdbcLockFactory.java | 318 ++++
.../iceberg/flink/maintenance/api/LockConfig.java | 260 +++
.../maintenance/api/MaintenanceTaskBuilder.java | 233 +++
.../flink/maintenance/api/RewriteDataFiles.java | 333 ++++
.../maintenance/api/RewriteDataFilesConfig.java | 176 ++
.../flink/maintenance/api/TableMaintenance.java | 443 +++++
.../iceberg/flink/maintenance/api/TaskResult.java | 64 +
.../iceberg/flink/maintenance/api/Trigger.java | 65 +
.../flink/maintenance/api/TriggerLockFactory.java | 64 +
.../flink/maintenance/api/ZKRetryPolicies.java | 36 +
.../flink/maintenance/api/ZkLockFactory.java | 271 +++
.../maintenance/operator/BaseCoordinator.java | 306 ++++
.../flink/maintenance/operator/DVPosition.java | 54 +
.../flink/maintenance/operator/DVWriteResult.java | 46 +
.../operator/DataFileRewriteCommitter.java | 210 +++
.../operator/DataFileRewritePlanner.java | 232 +++
.../operator/DataFileRewriteRunner.java | 271 +++
.../maintenance/operator/DeleteFilesProcessor.java | 121 ++
.../operator/EqualityConvertCommitter.java | 392 ++++
.../operator/EqualityConvertDVWriter.java | 354 ++++
.../operator/EqualityConvertPKIndex.java | 296 +++
.../maintenance/operator/EqualityConvertPlan.java | 81 +
.../operator/EqualityConvertPlanner.java | 783 ++++++++
.../operator/EqualityConvertReader.java | 258 +++
.../operator/EqualityDeleteFileScanTask.java | 51 +
.../operator/ExpireSnapshotsProcessor.java | 136 ++
.../flink/maintenance/operator/FileNameReader.java | 49 +
.../maintenance/operator/FileUriKeySelector.java | 60 +
.../operator/FlinkAddedRowsScanTask.java | 72 +
.../flink/maintenance/operator/IndexCommand.java | 109 ++
.../maintenance/operator/ListFileSystemFiles.java | 133 ++
.../maintenance/operator/ListMetadataFiles.java | 94 +
.../maintenance/operator/LockFactoryBuilder.java | 89 +
.../maintenance/operator/LockRegisterEvent.java | 47 +
.../maintenance/operator/LockReleaseEvent.java | 56 +
.../flink/maintenance/operator/LockRemover.java | 146 ++
.../operator/LockRemoverCoordinator.java | 60 +
.../maintenance/operator/LockRemoverOperator.java | 115 ++
.../operator/LockRemoverOperatorFactory.java | 86 +
.../flink/maintenance/operator/LogUtil.java | 26 +
.../maintenance/operator/MetadataTablePlanner.java | 133 ++
.../flink/maintenance/operator/MonitorSource.java | 206 +++
.../maintenance/operator/OrphanFilesDetector.java | 191 ++
.../flink/maintenance/operator/ReadCommand.java | 86 +
.../operator/SerializedEqualityValues.java | 51 +
.../operator/SingleThreadedIteratorSource.java | 197 ++
.../flink/maintenance/operator/SkipOnError.java | 90 +
.../maintenance/operator/StructLikeSerializer.java | 139 ++
.../flink/maintenance/operator/TableChange.java | 249 +++
.../operator/TableMaintenanceMetrics.java | 78 +
.../flink/maintenance/operator/TableReader.java | 120 ++
.../maintenance/operator/TaskResultAggregator.java | 104 ++
.../maintenance/operator/TriggerEvaluator.java | 151 ++
.../flink/maintenance/operator/TriggerManager.java | 308 ++++
.../operator/TriggerManagerCoordinator.java | 59 +
.../operator/TriggerManagerOperator.java | 322 ++++
.../operator/TriggerManagerOperatorFactory.java | 110 ++
.../flink/maintenance/operator/TriggerUtil.java | 46 +
.../sink/AvroGenericRecordToRowDataMapper.java | 61 +
.../iceberg/flink/sink/BaseDeltaTaskWriter.java | 128 ++
.../flink/sink/BucketPartitionKeySelector.java | 70 +
.../iceberg/flink/sink/BucketPartitioner.java | 103 ++
.../iceberg/flink/sink/BucketPartitionerUtil.java | 125 ++
.../iceberg/flink/sink/CachingTableSupplier.java | 91 +
.../apache/iceberg/flink/sink/CommitSummary.java | 108 ++
.../sink/CommittableToTableChangeConverter.java | 107 ++
.../apache/iceberg/flink/sink/DeltaManifests.java | 73 +
.../flink/sink/DeltaManifestsSerializer.java | 124 ++
.../flink/sink/EqualityFieldKeySelector.java | 88 +
.../iceberg/flink/sink/FlinkFileWriterFactory.java | 191 ++
.../iceberg/flink/sink/FlinkManifestUtil.java | 188 ++
.../org/apache/iceberg/flink/sink/FlinkSink.java | 786 ++++++++
.../iceberg/flink/sink/FlinkWriteResult.java | 40 +
.../iceberg/flink/sink/IcebergCommittable.java | 95 +
.../flink/sink/IcebergCommittableSerializer.java | 68 +
.../iceberg/flink/sink/IcebergCommitter.java | 337 ++++
.../iceberg/flink/sink/IcebergFilesCommitter.java | 490 +++++
.../flink/sink/IcebergFilesCommitterFactory.java | 72 +
.../flink/sink/IcebergFilesCommitterMetrics.java | 75 +
.../org/apache/iceberg/flink/sink/IcebergSink.java | 1161 ++++++++++++
.../iceberg/flink/sink/IcebergSinkBuilder.java | 103 ++
.../iceberg/flink/sink/IcebergSinkWriter.java | 113 ++
.../iceberg/flink/sink/IcebergStreamWriter.java | 119 ++
.../flink/sink/IcebergStreamWriterMetrics.java | 161 ++
.../iceberg/flink/sink/IcebergWriteAggregator.java | 155 ++
.../flink/sink/ManifestOutputFileFactory.java | 103 ++
.../iceberg/flink/sink/PartitionKeySelector.java | 66 +
.../iceberg/flink/sink/PartitionedDeltaWriter.java | 100 +
.../flink/sink/RowDataTaskWriterFactory.java | 277 +++
.../org/apache/iceberg/flink/sink/SinkUtil.java | 156 ++
.../iceberg/flink/sink/TaskWriterFactory.java | 45 +
.../flink/sink/UnpartitionedDeltaWriter.java | 72 +
.../iceberg/flink/sink/WriteResultSerializer.java | 63 +
.../flink/sink/dynamic/CompareSchemasVisitor.java | 299 +++
.../iceberg/flink/sink/dynamic/DataConverter.java | 235 +++
.../flink/sink/dynamic/DynamicCommittable.java | 100 +
.../sink/dynamic/DynamicCommittableSerializer.java | 106 ++
.../flink/sink/dynamic/DynamicCommitter.java | 427 +++++
.../sink/dynamic/DynamicCommitterMetrics.java | 50 +
.../flink/sink/dynamic/DynamicIcebergSink.java | 604 +++++++
.../iceberg/flink/sink/dynamic/DynamicRecord.java | 168 ++
.../flink/sink/dynamic/DynamicRecordGenerator.java | 34 +
.../flink/sink/dynamic/DynamicRecordInternal.java | 166 ++
.../dynamic/DynamicRecordInternalSerializer.java | 370 ++++
.../sink/dynamic/DynamicRecordInternalType.java | 96 +
.../flink/sink/dynamic/DynamicRecordProcessor.java | 240 +++
.../sink/dynamic/DynamicRecordWithConfig.java | 94 +
.../flink/sink/dynamic/DynamicSinkUtil.java | 87 +
.../sink/dynamic/DynamicTableRecordGenerator.java | 102 ++
.../sink/dynamic/DynamicTableUpdateOperator.java | 99 +
.../flink/sink/dynamic/DynamicWriteResult.java | 55 +
.../sink/dynamic/DynamicWriteResultAggregator.java | 245 +++
.../sink/dynamic/DynamicWriteResultSerializer.java | 64 +
.../iceberg/flink/sink/dynamic/DynamicWriter.java | 223 +++
.../flink/sink/dynamic/DynamicWriterMetrics.java | 74 +
.../flink/sink/dynamic/EvolveSchemaVisitor.java | 243 +++
.../flink/sink/dynamic/FlinkDynamicSinkConf.java | 120 ++
.../sink/dynamic/FlinkDynamicSinkOptions.java | 91 +
.../flink/sink/dynamic/HashKeyGenerator.java | 409 +++++
.../iceberg/flink/sink/dynamic/JobOperatorKey.java | 57 +
.../iceberg/flink/sink/dynamic/LRUCache.java | 64 +
.../flink/sink/dynamic/PartitionSpecEvolution.java | 137 ++
.../iceberg/flink/sink/dynamic/TableCreator.java | 34 +
.../iceberg/flink/sink/dynamic/TableKey.java | 79 +
.../flink/sink/dynamic/TableMetadataCache.java | 323 ++++
.../flink/sink/dynamic/TableSerializerCache.java | 141 ++
.../iceberg/flink/sink/dynamic/TableUpdater.java | 226 +++
.../VariantAvroDynamicTableRecordGenerator.java | 223 +++
.../iceberg/flink/sink/dynamic/WriteTarget.java | 132 ++
.../sink/shuffle/AggregatedStatisticsTracker.java | 264 +++
.../flink/sink/shuffle/CompletedStatistics.java | 128 ++
.../shuffle/CompletedStatisticsSerializer.java | 188 ++
.../iceberg/flink/sink/shuffle/DataStatistics.java | 48 +
.../sink/shuffle/DataStatisticsCoordinator.java | 536 ++++++
.../shuffle/DataStatisticsCoordinatorProvider.java | 70 +
.../flink/sink/shuffle/DataStatisticsOperator.java | 277 +++
.../shuffle/DataStatisticsOperatorFactory.java | 96 +
.../sink/shuffle/DataStatisticsSerializer.java | 204 +++
.../flink/sink/shuffle/GlobalStatistics.java | 114 ++
.../sink/shuffle/GlobalStatisticsSerializer.java | 199 ++
.../iceberg/flink/sink/shuffle/KeyAssignment.java | 155 ++
.../iceberg/flink/sink/shuffle/MapAssignment.java | 242 +++
.../flink/sink/shuffle/MapDataStatistics.java | 88 +
.../flink/sink/shuffle/MapRangePartitioner.java | 95 +
.../flink/sink/shuffle/RangePartitioner.java | 110 ++
.../sink/shuffle/RequestGlobalStatisticsEvent.java | 40 +
.../flink/sink/shuffle/SketchDataStatistics.java | 87 +
.../flink/sink/shuffle/SketchRangePartitioner.java | 51 +
.../iceberg/flink/sink/shuffle/SketchUtil.java | 161 ++
.../flink/sink/shuffle/SortKeySerializer.java | 408 +++++
.../sink/shuffle/SortKeySketchSerializer.java | 143 ++
.../iceberg/flink/sink/shuffle/SortKeyUtil.java | 58 +
.../flink/sink/shuffle/StatisticsEvent.java | 76 +
.../flink/sink/shuffle/StatisticsOrRecord.java | 132 ++
.../sink/shuffle/StatisticsOrRecordSerializer.java | 208 +++
.../shuffle/StatisticsOrRecordTypeInformation.java | 109 ++
.../iceberg/flink/sink/shuffle/StatisticsType.java | 55 +
.../iceberg/flink/sink/shuffle/StatisticsUtil.java | 143 ++
.../AvroGenericRecordFileScanTaskReader.java | 42 +
.../apache/iceberg/flink/source/DataIterator.java | 158 ++
.../iceberg/flink/source/DataTaskReader.java | 47 +
.../iceberg/flink/source/FileScanTaskReader.java | 35 +
.../iceberg/flink/source/FlinkInputFormat.java | 141 ++
.../iceberg/flink/source/FlinkInputSplit.java | 48 +
.../apache/iceberg/flink/source/FlinkSource.java | 318 ++++
.../iceberg/flink/source/FlinkSplitPlanner.java | 189 ++
.../apache/iceberg/flink/source/IcebergSource.java | 705 ++++++++
.../iceberg/flink/source/IcebergTableSource.java | 240 +++
.../flink/source/RowDataFileScanTaskReader.java | 162 ++
.../iceberg/flink/source/RowDataRewriter.java | 166 ++
.../RowDataToAvroGenericRecordConverter.java | 70 +
.../apache/iceberg/flink/source/ScanContext.java | 601 ++++++
.../apache/iceberg/flink/source/SourceUtil.java | 77 +
.../flink/source/StreamingMonitorFunction.java | 270 +++
.../flink/source/StreamingReaderOperator.java | 250 +++
.../flink/source/StreamingStartingStrategy.java | 61 +
.../source/assigner/DefaultSplitAssigner.java | 119 ++
.../flink/source/assigner/GetSplitResult.java | 77 +
.../assigner/OrderedSplitAssignerFactory.java | 46 +
.../assigner/SimpleSplitAssignerFactory.java | 37 +
.../flink/source/assigner/SplitAssigner.java | 124 ++
.../source/assigner/SplitAssignerFactory.java | 30 +
.../flink/source/assigner/SplitAssignerType.java | 33 +
.../enumerator/AbstractIcebergEnumerator.java | 188 ++
.../enumerator/ContinuousEnumerationResult.java | 57 +
.../enumerator/ContinuousIcebergEnumerator.java | 188 ++
.../source/enumerator/ContinuousSplitPlanner.java | 30 +
.../enumerator/ContinuousSplitPlannerImpl.java | 248 +++
.../source/enumerator/EnumerationHistory.java | 100 +
.../enumerator/IcebergEnumeratorPosition.java | 79 +
.../IcebergEnumeratorPositionSerializer.java | 90 +
.../source/enumerator/IcebergEnumeratorState.java | 65 +
.../IcebergEnumeratorStateSerializer.java | 194 ++
.../source/enumerator/StaticIcebergEnumerator.java | 51 +
.../flink/source/reader/ArrayBatchRecords.java | 179 ++
.../reader/ArrayPoolDataIteratorBatcher.java | 161 ++
.../source/reader/AvroGenericRecordConverter.java | 69 +
.../reader/AvroGenericRecordReaderFunction.java | 110 ++
.../reader/ColumnStatsWatermarkExtractor.java | 111 ++
.../source/reader/ConverterReaderFunction.java | 117 ++
.../flink/source/reader/DataIteratorBatcher.java | 36 +
.../source/reader/DataIteratorReaderFunction.java | 43 +
.../flink/source/reader/IcebergSourceReader.java | 77 +
.../source/reader/IcebergSourceReaderMetrics.java | 61 +
.../source/reader/IcebergSourceSplitReader.java | 178 ++
.../flink/source/reader/LimitableDataIterator.java | 56 +
.../flink/source/reader/ListBatchRecords.java | 85 +
.../source/reader/ListDataIteratorBatcher.java | 94 +
.../source/reader/MetaDataReaderFunction.java | 65 +
.../flink/source/reader/PoolWithWakeup.java | 108 ++
.../flink/source/reader/ReaderFunction.java | 31 +
.../flink/source/reader/RecordAndPosition.java | 79 +
.../iceberg/flink/source/reader/RecordFactory.java | 34 +
.../iceberg/flink/source/reader/RecordLimiter.java | 45 +
.../iceberg/flink/source/reader/RowConverter.java | 64 +
.../flink/source/reader/RowDataConverter.java | 32 +
.../flink/source/reader/RowDataReaderFunction.java | 115 ++
.../flink/source/reader/RowDataRecordFactory.java | 74 +
.../source/reader/SerializableRecordEmitter.java | 40 +
.../source/reader/SplitWatermarkExtractor.java | 28 +
.../flink/source/reader/WakeableIterator.java | 35 +
.../reader/WatermarkExtractorRecordEmitter.java | 70 +
.../flink/source/split/IcebergSourceSplit.java | 221 +++
.../source/split/IcebergSourceSplitSerializer.java | 64 +
.../source/split/IcebergSourceSplitState.java | 37 +
.../source/split/IcebergSourceSplitStatus.java | 25 +
.../flink/source/split/SerializableComparator.java | 24 +
.../flink/source/split/SplitComparators.java | 78 +
.../flink/source/split/SplitRequestEvent.java | 54 +
.../iceberg/flink/util/ElapsedTimeGauge.java | 47 +
.../iceberg/flink/util/FlinkAlterTableUtil.java | 260 +++
.../iceberg/flink/util/FlinkCompatibilityUtil.java | 56 +
.../apache/iceberg/flink/util/FlinkPackage.java | 61 +
.../iceberg/flink/util/SerializerHelper.java | 209 +++
.../org.apache.flink.table.factories.Factory | 17 +
.../flink/AvroGenericRecordConverterBase.java | 90 +
.../org/apache/iceberg/flink/CatalogTestBase.java | 122 ++
.../org/apache/iceberg/flink/DataGenerator.java | 45 +
.../org/apache/iceberg/flink/DataGenerators.java | 1585 ++++++++++++++++
.../iceberg/flink/HadoopCatalogExtension.java | 105 ++
.../apache/iceberg/flink/HadoopTableExtension.java | 59 +
.../iceberg/flink/MiniFlinkClusterExtension.java | 67 +
.../org/apache/iceberg/flink/RowDataConverter.java | 160 ++
.../org/apache/iceberg/flink/SimpleDataUtil.java | 484 +++++
.../java/org/apache/iceberg/flink/SqlBase.java | 110 ++
.../java/org/apache/iceberg/flink/TestBase.java | 145 ++
.../apache/iceberg/flink/TestCatalogLoader.java | 116 ++
.../iceberg/flink/TestCatalogTableLoader.java | 113 ++
.../apache/iceberg/flink/TestChangeLogTable.java | 296 +++
.../iceberg/flink/TestDataFileSerialization.java | 205 +++
.../org/apache/iceberg/flink/TestFixtures.java | 61 +
.../iceberg/flink/TestFlinkAnonymousTable.java | 65 +
.../iceberg/flink/TestFlinkCatalogDatabase.java | 253 +++
.../iceberg/flink/TestFlinkCatalogFactory.java | 119 ++
.../iceberg/flink/TestFlinkCatalogTable.java | 788 ++++++++
.../flink/TestFlinkCatalogTablePartitions.java | 119 ++
.../apache/iceberg/flink/TestFlinkConfParser.java | 61 +
.../org/apache/iceberg/flink/TestFlinkFilters.java | 467 +++++
.../apache/iceberg/flink/TestFlinkHiveCatalog.java | 101 ++
.../apache/iceberg/flink/TestFlinkReadConf.java | 116 ++
.../apache/iceberg/flink/TestFlinkSchemaUtil.java | 474 +++++
.../apache/iceberg/flink/TestFlinkTableSink.java | 321 ++++
.../flink/TestFlinkTableSinkCompaction.java | 217 +++
.../iceberg/flink/TestFlinkTableSinkExtended.java | 401 ++++
.../apache/iceberg/flink/TestFlinkUnknownType.java | 179 ++
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 342 ++++
.../apache/iceberg/flink/TestFlinkUuidType.java | 192 ++
.../flink/TestFlinkVariantShreddingType.java | 985 ++++++++++
.../apache/iceberg/flink/TestFlinkVariantType.java | 128 ++
.../java/org/apache/iceberg/flink/TestHelpers.java | 758 ++++++++
.../apache/iceberg/flink/TestIcebergConnector.java | 549 ++++++
.../flink/TestManifestFileSerialization.java | 173 ++
.../apache/iceberg/flink/TestRowDataWrapper.java | 94 +
.../org/apache/iceberg/flink/TestTableLoader.java | 57 +
.../iceberg/flink/TestTableSerialization.java | 110 ++
.../flink/actions/TestRewriteDataFilesAction.java | 528 ++++++
.../apache/iceberg/flink/data/RandomRowData.java | 38 +
.../iceberg/flink/data/RowDataToRowMapper.java | 50 +
.../flink/data/TestFlinkAvroReaderWriter.java | 141 ++
.../iceberg/flink/data/TestFlinkFormatModel.java | 86 +
.../flink/data/TestFlinkOrcReaderWriter.java | 123 ++
.../iceberg/flink/data/TestFlinkParquetReader.java | 284 +++
.../iceberg/flink/data/TestFlinkParquetWriter.java | 128 ++
.../iceberg/flink/data/TestFlinkVariants.java | 183 ++
.../iceberg/flink/data/TestRowDataProjection.java | 598 ++++++
.../iceberg/flink/data/TestRowProjection.java | 582 ++++++
.../iceberg/flink/data/TestStructRowData.java | 100 +
.../flink/data/TestVariantRowDataWrapper.java | 292 +++
.../api/MaintenanceTaskInfraExtension.java | 78 +
.../maintenance/api/MaintenanceTaskTestBase.java | 120 ++
.../api/TestConvertEqualityDeletes.java | 1404 +++++++++++++++
.../api/TestConvertEqualityDeletesConfig.java | 69 +
.../api/TestConvertEqualityDeletesE2E.java | 196 ++
.../maintenance/api/TestDeleteOrphanFiles.java | 340 ++++
.../api/TestDeleteOrphanFilesConfig.java | 91 +
.../flink/maintenance/api/TestExpireSnapshots.java | 233 +++
.../maintenance/api/TestExpireSnapshotsConfig.java | 93 +
.../flink/maintenance/api/TestJdbcLockFactory.java | 106 ++
.../flink/maintenance/api/TestLockFactoryBase.java | 93 +
.../flink/maintenance/api/TestMaintenanceE2E.java | 152 ++
.../maintenance/api/TestRewriteDataFiles.java | 695 +++++++
.../api/TestRewriteDataFilesConfig.java | 142 ++
.../maintenance/api/TestTableMaintenance.java | 512 ++++++
.../api/TestTableMaintenanceCoordinationLock.java | 344 ++++
.../flink/maintenance/api/TestZkLockFactory.java | 128 ++
.../flink/maintenance/operator/CollectingSink.java | 116 ++
.../maintenance/operator/CoordinatorTestBase.java | 43 +
.../flink/maintenance/operator/ManualSource.java | 320 ++++
.../operator/MetricsReporterFactoryForTests.java | 192 ++
.../maintenance/operator/OperatorTestBase.java | 545 ++++++
.../flink/maintenance/operator/RewriteUtil.java | 94 +
.../operator/TestDataFileRewriteCommitter.java | 281 +++
.../operator/TestDataFileRewritePlanner.java | 301 ++++
.../operator/TestDataFileRewriteRunner.java | 379 ++++
.../operator/TestDeleteFilesProcessor.java | 115 ++
.../operator/TestEqualityConvertCommitter.java | 678 +++++++
.../operator/TestEqualityConvertDVWriter.java | 485 +++++
.../operator/TestEqualityConvertPKIndex.java | 747 ++++++++
.../operator/TestEqualityConvertPlanner.java | 1155 ++++++++++++
.../operator/TestEqualityConvertReader.java | 421 +++++
.../operator/TestExpireSnapshotsProcessor.java | 122 ++
.../maintenance/operator/TestFlinkPojoTypes.java | 47 +
.../operator/TestListFileSystemFiles.java | 110 ++
.../operator/TestListMetadataFiles.java | 125 ++
.../flink/maintenance/operator/TestLockConfig.java | 84 +
.../operator/TestLockFactoryBuilder.java | 109 ++
.../operator/TestLockRemoveCoordinator.java | 69 +
.../maintenance/operator/TestLockRemover.java | 444 +++++
.../operator/TestLockRemoverOperation.java | 228 +++
.../maintenance/operator/TestMonitorSource.java | 360 ++++
.../operator/TestOrphanFilesDetector.java | 246 +++
.../maintenance/operator/TestSkipOnError.java | 100 +
.../operator/TestStructLikeSerializer.java | 164 ++
.../operator/TestTablePlanerAndReader.java | 120 ++
.../operator/TestTaskResultAggregator.java | 78 +
.../maintenance/operator/TestTriggerManager.java | 671 +++++++
.../operator/TestTriggerManagerCoordinator.java | 102 ++
.../operator/TestTriggerManagerOperator.java | 668 +++++++
.../apache/iceberg/flink/sink/SinkTestUtil.java | 62 +
.../sink/TestAvroGenericRecordToRowDataMapper.java | 38 +
.../flink/sink/TestBucketPartitionKeySelector.java | 67 +
.../iceberg/flink/sink/TestBucketPartitioner.java | 108 ++
.../TestBucketPartitionerFlinkIcebergSink.java | 233 +++
.../flink/sink/TestBucketPartitionerUtil.java | 126 ++
.../flink/sink/TestCachingTableSupplier.java | 81 +
.../TestCommittableToTableChangeConverter.java | 324 ++++
.../flink/sink/TestCompressionSettings.java | 260 +++
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 434 +++++
.../flink/sink/TestFlinkFileWriterFactory.java | 65 +
.../iceberg/flink/sink/TestFlinkIcebergSink.java | 143 ++
.../flink/sink/TestFlinkIcebergSinkBase.java | 133 ++
.../flink/sink/TestFlinkIcebergSinkBranch.java | 158 ++
.../sink/TestFlinkIcebergSinkDistributionMode.java | 658 +++++++
.../flink/sink/TestFlinkIcebergSinkExtended.java | 250 +++
...FlinkIcebergSinkRangeDistributionBucketing.java | 268 +++
.../iceberg/flink/sink/TestFlinkIcebergSinkV2.java | 256 +++
.../flink/sink/TestFlinkIcebergSinkV2Base.java | 424 +++++
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 151 ++
.../TestFlinkIcebergSinkV2DistributionMode.java | 707 ++++++++
.../iceberg/flink/sink/TestFlinkManifest.java | 315 ++++
.../flink/sink/TestFlinkPartitioningWriters.java | 76 +
.../flink/sink/TestFlinkPositionDeltaWriters.java | 65 +
.../flink/sink/TestFlinkRollingFileWriters.java | 50 +
.../iceberg/flink/sink/TestFlinkWriterMetrics.java | 67 +
.../iceberg/flink/sink/TestIcebergCommitter.java | 1487 +++++++++++++++
.../flink/sink/TestIcebergFilesCommitter.java | 1225 +++++++++++++
.../apache/iceberg/flink/sink/TestIcebergSink.java | 563 ++++++
.../iceberg/flink/sink/TestIcebergSinkBranch.java | 142 ++
.../sink/TestIcebergSinkTableMaintenance.java | 459 +++++
.../iceberg/flink/sink/TestIcebergSinkV2.java | 289 +++
.../flink/sink/TestIcebergSinkV2Branch.java | 110 ++
.../flink/sink/TestIcebergStreamWriter.java | 409 +++++
.../flink/sink/TestIcebergStreamWriterMetrics.java | 42 +
.../flink/sink/TestManifestOutputFileFactory.java | 112 ++
.../flink/sink/TestRowDataPartitionKey.java | 298 +++
.../apache/iceberg/flink/sink/TestTaskWriters.java | 241 +++
.../DynamicRecordInternalSerializerTestBase.java | 100 +
.../sink/dynamic/TestCompareSchemasVisitor.java | 386 ++++
.../dynamic/TestDynamicCommittableSerializer.java | 114 ++
.../flink/sink/dynamic/TestDynamicCommitter.java | 1164 ++++++++++++
.../flink/sink/dynamic/TestDynamicIcebergSink.java | 1905 ++++++++++++++++++++
.../sink/dynamic/TestDynamicIcebergSinkPerf.java | 244 +++
.../TestDynamicRecordInternalSerializer.java | 119 ++
...DynamicRecordInternalSerializerWriteSchema.java | 28 +
...namicRecordInternalSerializerWriteSchemaId.java | 28 +
...cordInternalSerializerWriteSchemaIdLongUTF.java | 28 +
...RecordInternalSerializerWriteSchemaLongUTF.java | 28 +
.../sink/dynamic/TestDynamicRecordProcessor.java | 101 ++
.../sink/dynamic/TestDynamicRecordWithConfig.java | 120 ++
.../dynamic/TestDynamicTableUpdateOperator.java | 244 +++
.../dynamic/TestDynamicWriteResultAggregator.java | 259 +++
.../dynamic/TestDynamicWriteResultSerializer.java | 78 +
.../flink/sink/dynamic/TestDynamicWriter.java | 378 ++++
.../sink/dynamic/TestEvolveSchemaVisitor.java | 834 +++++++++
.../flink/sink/dynamic/TestHashKeyGenerator.java | 715 ++++++++
.../iceberg/flink/sink/dynamic/TestLRUCache.java | 88 +
.../sink/dynamic/TestPartitionSpecEvolution.java | 188 ++
.../flink/sink/dynamic/TestRowDataConverter.java | 294 +++
.../flink/sink/dynamic/TestTableMetadataCache.java | 220 +++
.../sink/dynamic/TestTableSerializerCache.java | 167 ++
.../flink/sink/dynamic/TestTableUpdater.java | 434 +++++
...TestVariantAvroDynamicTableRecordGenerator.java | 502 ++++++
.../flink/sink/shuffle/DataDistributionUtil.java | 178 ++
.../iceberg/flink/sink/shuffle/Fixtures.java | 101 ++
.../shuffle/TestAggregatedStatisticsTracker.java | 465 +++++
.../shuffle/TestCompletedStatisticsSerializer.java | 103 ++
.../sink/shuffle/TestDataDistributionUtil.java | 49 +
.../shuffle/TestDataStatisticsCoordinator.java | 373 ++++
.../TestDataStatisticsCoordinatorProvider.java | 187 ++
.../sink/shuffle/TestDataStatisticsOperator.java | 380 ++++
.../sink/shuffle/TestDataStatisticsSerializer.java | 53 +
.../shuffle/TestGlobalStatisticsSerializer.java | 59 +
.../flink/sink/shuffle/TestMapDataStatistics.java | 67 +
.../sink/shuffle/TestMapRangePartitioner.java | 436 +++++
.../flink/sink/shuffle/TestRangePartitioner.java | 65 +
.../sink/shuffle/TestRangePartitionerSkew.java | 183 ++
.../sink/shuffle/TestSketchDataStatistics.java | 60 +
.../sink/shuffle/TestSketchRangePartitioner.java | 88 +
.../iceberg/flink/sink/shuffle/TestSketchUtil.java | 200 ++
.../sink/shuffle/TestSortKeySerializerBase.java | 65 +
.../shuffle/TestSortKeySerializerNestedStruct.java | 55 +
.../shuffle/TestSortKeySerializerPrimitives.java | 93 +
.../shuffle/TestSortKeySerializerSnapshot.java | 235 +++
.../flink/sink/shuffle/TestSortKeyUtil.java | 73 +
.../shuffle/TestStatisticsOrRecordSerializer.java | 72 +
.../TestStatisticsOrRecordTypeInformation.java | 46 +
.../iceberg/flink/source/BoundedTableFactory.java | 184 ++
.../iceberg/flink/source/BoundedTestSource.java | 108 ++
.../flink/source/ChangeLogTableTestBase.java | 95 +
.../apache/iceberg/flink/source/SplitHelpers.java | 200 ++
.../apache/iceberg/flink/source/SqlHelpers.java | 60 +
.../iceberg/flink/source/TableSourceTestBase.java | 105 ++
.../flink/source/TestBoundedTableFactory.java | 81 +
.../iceberg/flink/source/TestFlinkInputFormat.java | 211 +++
.../source/TestFlinkInputFormatReaderDeletes.java | 71 +
.../flink/source/TestFlinkMergingMetrics.java | 67 +
.../flink/source/TestFlinkMetaDataTable.java | 816 +++++++++
.../flink/source/TestFlinkReaderDeletesBase.java | 90 +
.../apache/iceberg/flink/source/TestFlinkScan.java | 540 ++++++
.../iceberg/flink/source/TestFlinkScanSql.java | 69 +
.../iceberg/flink/source/TestFlinkSource.java | 90 +
.../flink/source/TestFlinkSourceConfig.java | 61 +
.../iceberg/flink/source/TestFlinkSourceSql.java | 87 +
.../iceberg/flink/source/TestFlinkTableSource.java | 561 ++++++
.../flink/source/TestIcebergSourceBounded.java | 151 ++
.../TestIcebergSourceBoundedConverterBase.java | 223 +++
.../TestIcebergSourceBoundedGenericRecord.java | 96 +
.../flink/source/TestIcebergSourceBoundedRow.java | 58 +
.../flink/source/TestIcebergSourceBoundedSql.java | 76 +
.../flink/source/TestIcebergSourceContinuous.java | 573 ++++++
.../flink/source/TestIcebergSourceFailover.java | 397 ++++
...cebergSourceFailoverWithWatermarkExtractor.java | 130 ++
.../source/TestIcebergSourceInferParallelism.java | 181 ++
.../source/TestIcebergSourceReaderDeletes.java | 102 ++
.../iceberg/flink/source/TestIcebergSourceSql.java | 242 +++
.../TestIcebergSourceWithWatermarkExtractor.java | 470 +++++
.../TestIcebergSpeculativeExecutionSupport.java | 206 +++
.../source/TestMetadataTableReadableMetrics.java | 366 ++++
.../flink/source/TestProjectMetaColumn.java | 189 ++
.../TestRowDataToAvroGenericRecordConverter.java | 36 +
.../iceberg/flink/source/TestScanContext.java | 113 ++
.../iceberg/flink/source/TestSourceUtil.java | 61 +
.../apache/iceberg/flink/source/TestSqlBase.java | 173 ++
.../iceberg/flink/source/TestStreamScanSql.java | 490 +++++
.../flink/source/TestStreamingMonitorFunction.java | 399 ++++
.../flink/source/TestStreamingReaderOperator.java | 288 +++
.../source/assigner/SplitAssignerTestBase.java | 132 ++
.../source/assigner/TestDefaultSplitAssigner.java | 43 +
.../TestFileSequenceNumberBasedSplitAssigner.java | 81 +
.../assigner/TestWatermarkBasedSplitAssigner.java | 143 ++
.../enumerator/ManualContinuousSplitPlanner.java | 97 +
.../TestContinuousIcebergEnumerator.java | 352 ++++
.../enumerator/TestContinuousSplitPlannerImpl.java | 734 ++++++++
...estContinuousSplitPlannerImplStartStrategy.java | 219 +++
.../source/enumerator/TestEnumerationHistory.java | 135 ++
.../TestIcebergEnumeratorStateSerializer.java | 146 ++
.../source/reader/ReaderFunctionTestBase.java | 214 +++
.../iceberg/flink/source/reader/ReaderUtil.java | 139 ++
.../flink/source/reader/TestArrayBatchRecords.java | 69 +
.../TestArrayPoolDataIteratorBatcherRowData.java | 541 ++++++
.../TestArrayPoolDataIteratorBatcherWakeup.java | 175 ++
.../reader/TestColumnStatsWatermarkExtractor.java | 198 ++
.../source/reader/TestIcebergSourceReader.java | 177 ++
.../source/reader/TestLimitableDataIterator.java | 82 +
.../flink/source/reader/TestPoolWithWakeup.java | 122 ++
.../source/reader/TestRowDataReaderFunction.java | 69 +
.../TestWatermarkExtractorRecordEmitter.java | 164 ++
.../flink/source/reader/TestingMetricGroup.java | 102 ++
.../split/TestIcebergSourceSplitSerializer.java | 183 ++
.../iceberg/flink/util/TestFlinkPackage.java | 55 +
...he.flink.metrics.reporter.MetricReporterFactory | 16 +
.../org.apache.flink.table.factories.Factory | 16 +
558 files changed, 115365 insertions(+)
diff --git a/flink/v2.2/build.gradle b/flink/v2.2/build.gradle
new file mode 100644
index 0000000000..97efc0b22b
--- /dev/null
+++ b/flink/v2.2/build.gradle
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+String flinkMajorVersion = '2.1'
+String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
+
+project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
+
+ dependencies {
+ implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+ api project(':iceberg-api')
+ implementation project(':iceberg-common')
+ implementation project(':iceberg-core')
+ api project(':iceberg-data')
+ implementation project(':iceberg-orc')
+ implementation project(':iceberg-parquet')
+ implementation project(':iceberg-hive-metastore')
+
+ compileOnly libs.flink21.avro
+ compileOnly libs.joda.time
+ // dropwizard histogram metrics (optional in Flink)
+ compileOnly libs.flink21.metrics.dropwizard
+ compileOnly libs.flink21.streaming.java
+ compileOnly "${libs.flink21.streaming.java.get().module}:${libs.flink21.streaming.java.get().getVersion()}:tests"
+ compileOnly libs.flink21.table.api.java.bridge
+ compileOnly "org.apache.flink:flink-table-planner_${scalaVersion}:${libs.versions.flink21.get()}"
+ compileOnly libs.flink21.connector.base
+ compileOnly libs.flink21.connector.files
+
+ compileOnly libs.hadoop3.hdfs
+ compileOnly libs.hadoop3.common
+ compileOnly(libs.hadoop3.minicluster) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ }
+
+ implementation(libs.parquet.avro) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ // already shaded by Parquet
+ exclude group: 'it.unimi.dsi'
+ exclude group: 'org.codehaus.jackson'
+ }
+
+ compileOnly libs.avro.avro
+
+ implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
+ exclude group: 'org.apache.hadoop'
+ exclude group: 'commons-lang'
+ // These artifacts are shaded and included in the orc-core fat jar
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.hive', module: 'hive-storage-api'
+ exclude group: 'org.slf4j'
+ }
+
+ implementation libs.datasketches
+ implementation libs.roaringbitmap
+
+ testImplementation libs.flink21.connector.test.utils
+ testImplementation libs.flink21.core
+ testImplementation libs.flink21.runtime
+ testImplementation(libs.flink21.test.utilsjunit) {
+ exclude group: 'junit'
+ }
+ testImplementation(libs.flink21.test.utils) {
+ exclude group: "org.apache.curator", module: 'curator-test'
+ exclude group: 'junit'
+ }
+
+ testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-orc', configuration: 'testArtifacts')
+ testImplementation(testFixtures(project(':iceberg-parquet')))
+
+ // By default, hive-exec is a fat/uber jar and it exports a guava library
+ // that's really old. We use the core classifier to be able to override our guava
+ // version. Luckily, hive-exec seems to work okay so far with this version of guava
+ // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
+ testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hive', module: 'hive-llap-tez'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.calcite'
+ exclude group: 'org.apache.calcite.avatica'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+
+ testImplementation(libs.hive2.metastore) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hbase'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'co.cask.tephra'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+ exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+ exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+ exclude group: 'com.tdunning', module: 'json'
+ exclude group: 'javax.transaction', module: 'transaction-api'
+ exclude group: 'com.zaxxer', module: 'HikariCP'
+ exclude group: 'org.slf4j'
+ }
+
+ testImplementation libs.awaitility
+ testImplementation libs.assertj.core
+ testImplementation libs.sqlite.jdbc
+ }
+
+ test {
+ useJUnitPlatform()
+ }
+}
+
+project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
+ apply plugin: 'com.gradleup.shadow'
+
+ tasks.jar.dependsOn tasks.shadowJar
+
+ sourceSets {
+ integration {
+ java.srcDir "$projectDir/src/integration/java"
+ resources.srcDir "$projectDir/src/integration/resources"
+ }
+ }
+
+ configurations {
+ implementation {
+ // included in Flink
+ exclude group: 'org.slf4j'
+ exclude group: 'org.apache.commons'
+ exclude group: 'commons-pool'
+ exclude group: 'commons-codec'
+ exclude group: 'org.xerial.snappy'
+ exclude group: 'javax.xml.bind'
+ exclude group: 'javax.annotation'
+ }
+ }
+
+ dependencies {
+ implementation(project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}")) {
+ exclude group: 'org.apache.flink'
+ }
+ implementation project(':iceberg-aws')
+ implementation project(':iceberg-azure')
+ implementation(project(':iceberg-aliyun')) {
+ exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
+ exclude group: 'org.apache.httpcomponents', module: 'httpclient'
+ exclude group: 'commons-logging', module: 'commons-logging'
+ }
+ implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
+ implementation(project(':iceberg-nessie')) {
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+
+ // for integration testing with the flink-runtime-jar
+ // all of those dependencies are required because the integration test extends FlinkTestBase
+ integrationCompileOnly project(':iceberg-api')
+ integrationImplementation libs.assertj.core
+ integrationImplementation project(path: ":iceberg-flink:iceberg-flink-${flinkMajorVersion}", configuration: "testArtifacts")
+ integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ integrationImplementation(libs.flink21.test.utils) {
+ exclude group: "org.apache.curator", module: 'curator-test'
+ exclude group: 'junit'
+ }
+
+ integrationImplementation libs.flink21.table.api.java.bridge
+ integrationImplementation "org.apache.flink:flink-table-planner_${scalaVersion}:${libs.versions.flink21.get()}"
+
+ integrationImplementation libs.hadoop3.common
+ integrationImplementation libs.hadoop3.hdfs
+ integrationImplementation(libs.hadoop3.minicluster) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ }
+
+ integrationImplementation(libs.hive2.metastore) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hbase'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'co.cask.tephra'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+ exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+ exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+ exclude group: 'com.tdunning', module: 'json'
+ exclude group: 'javax.transaction', module: 'transaction-api'
+ exclude group: 'com.zaxxer', module: 'HikariCP'
+ exclude group: 'org.slf4j'
+ }
+
+ integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ exclude group: 'org.pentaho' // missing dependency
+ exclude group: 'org.apache.hive', module: 'hive-llap-tez'
+ exclude group: 'org.apache.logging.log4j'
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.calcite'
+ exclude group: 'org.apache.calcite.avatica'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+ }
+
+ shadowJar {
+ configurations = [project.configurations.runtimeClasspath]
+
+ zip64 true
+
+ // Relocate dependencies to avoid conflicts
+ relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
+ relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
+ relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
+ relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
+ relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
+ relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
+ relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
+ relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
+ relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
+ relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
+ relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
+ relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
+ relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
+
+ archiveClassifier.set(null)
+ }
+
+ task integrationTest(type: Test) {
+ description = "Test Flink Runtime Jar against Flink ${flinkMajorVersion}"
+ group = "verification"
+ jvmArgs += project.property('extraJvmArgs')
+ testClassesDirs = sourceSets.integration.output.classesDirs
+ classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
+ inputs.file(shadowJar.archiveFile.get().asFile.path)
+ }
+ integrationTest.dependsOn shadowJar
+ check.dependsOn integrationTest
+
+ jar {
+ enabled = false
+ }
+
+ apply from: "${rootDir}/runtime-deps.gradle"
+}
diff --git a/flink/v2.2/flink-runtime/LICENSE b/flink/v2.2/flink-runtime/LICENSE
new file mode 100644
index 0000000000..3ba5bdbe5f
--- /dev/null
+++ b/flink/v2.2/flink-runtime/LICENSE
@@ -0,0 +1,1047 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This product bundles and includes code from Apache Avro.
+
+* Conversion in DecimalWriter is based on Avro's Conversions.DecimalConversion.
+
+Copyright: 2014-2020 The Apache Software Foundation.
+Project URL: https://avro.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Jackson JSON Processor.
+
+Copyright: 2007-2020 Tatu Saloranta and other contributors
+Project URL: https://github.com/FasterXML/jackson
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles FastDoubleParser (via Jackson JSON Processor).
+
+Copyright: 2023 Werner Randelshofer, Switzerland
+Project URL: https://github.com/wrandelshofer/FastDoubleParser
+License: MIT
+
+| Copyright (c) 2023 Werner Randelshofer, Switzerland
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product bundles fast_float (bundled by FastDoubleParser).
+
+Copyright: 2021 The fast_float authors
+Project URL: https://github.com/fastfloat/fast_float
+License: MIT
+
+| Copyright (c) 2021 The fast_float authors
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product bundles bigint (bundled by FastDoubleParser).
+
+Copyright: 2022 Tim Buktu
+Project URL: https://github.com/tbuktu/bigint
+License: BSD 2-Clause
+
+| Copyright 2022 Tim Buktu
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions
+| are met:
+|
+| 1. Redistributions of source code must retain the above copyright notice, this
+| list of conditions and the following disclaimer.
+|
+| 2. Redistributions in binary form must reproduce the above copyright notice,
+| this list of conditions and the following disclaimer in the documentation
+| and/or other materials provided with the distribution.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+| ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+| WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+| DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+| FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This product bundles and includes code from Apache Parquet.
+
+* DynMethods.java
+* DynConstructors.java
+* IOUtil.java readFully
+* ByteBufferInputStream implementations
+
+Copyright: 2014-2020 The Apache Software Foundation.
+Project URL: https://parquet.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Apache Thrift (bundled by Parquet).
+
+Copyright: 2006-2010 The Apache Software Foundation.
+Project URL: https://thrift.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from Daniel Lemire's JavaFastPFOR project (bundled by Parquet).
+
+Copyright: 2013 Daniel Lemire
+Project URL: https://github.com/lemire/JavaFastPFOR
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles fastutil (bundled by Parquet).
+
+Copyright: 2002-2014 Sebastiano Vigna
+Project URL: http://fastutil.di.unimi.it/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Zero-Allocation Hashing (bundled by Parquet).
+
+Project URL: https://github.com/OpenHFT/Zero-Allocation-Hashing
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Apache ORC.
+
+Copyright: 2013-2020 The Apache Software Foundation.
+Project URL: https://orc.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Apache Hive's Storage API (bundled by ORC).
+
+Copyright: 2008-2020 The Apache Software Foundation.
+Project URL: https://hive.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Google protobuf (bundled by ORC).
+
+Copyright: 2008 Google Inc.
+Project URL: https://developers.google.com/protocol-buffers
+License: BSD 3-Clause
+
+| Copyright 2008 Google Inc. All rights reserved.
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions are
+| met:
+|
+| * Redistributions of source code must retain the above copyright
+| notice, this list of conditions and the following disclaimer.
+| * Redistributions in binary form must reproduce the above
+| copyright notice, this list of conditions and the following disclaimer
+| in the documentation and/or other materials provided with the
+| distribution.
+| * Neither the name of Google Inc. nor the names of its
+| contributors may be used to endorse or promote products derived from
+| this software without specific prior written permission.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+|
+| Code generated by the Protocol Buffer compiler is owned by the owner
+| of the input file used when generating it. This code is not
+| standalone and requires a support library to be linked with it. This
+| support library is itself covered by the above license.
+
+--------------------------------------------------------------------------------
+
+This product bundles Apache Datasketches.
+
+Project URL: https://datasketches.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Airlift Aircompressor.
+
+Copyright: 2011-2020 Aircompressor authors.
+Project URL: https://github.com/airlift/aircompressor
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from Cloudera Kite.
+
+* SchemaVisitor and visit methods
+
+Copyright: 2013-2017 Cloudera Inc.
+Project URL: https://kitesdk.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from Presto.
+
+* Retry wait and jitter logic in Tasks.java
+
+Copyright: 2016 Facebook and contributors
+Project URL: https://prestodb.io/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Google Guava.
+
+Copyright: 2006-2020 The Guava Authors
+Project URL: https://github.com/google/guava
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Google Error Prone Annotations.
+
+Copyright: Copyright 2011-2019 The Error Prone Authors
+Project URL: https://github.com/google/error-prone
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles JSpecify.
+
+Project URL: https://github.com/jspecify/jspecify
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Caffeine by Ben Manes.
+
+Copyright: 2014-2020 Ben Manes and contributors
+Project URL: https://github.com/ben-manes/caffeine
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles ThreeTen Extra.
+
+Copyright: 2007-present, Stephen Colebourne & Michael Nascimento Santos.
+Project URL: https://www.threeten.org/threeten-extra/
+License: BSD 3-Clause
+
+| Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos.
+| All rights reserved.
+|
+| * Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions are met:
+|
+| * Redistributions of source code must retain the above copyright notice,
+| this list of conditions and the following disclaimer.
+|
+| * Redistributions in binary form must reproduce the above copyright notice,
+| this list of conditions and the following disclaimer in the documentation
+| and/or other materials provided with the distribution.
+|
+| * Neither the name of JSR-310 nor the names of its contributors
+| may be used to endorse or promote products derived from this software
+| without specific prior written permission.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+| CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This product bundles Project Nessie.
+
+Copyright: Copyright 2015-2025 Dremio Corporation
+Project URL: https://projectnessie.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Flink.
+
+* Primary key validation logic in FlinkSchemaUtil.java
+* Avro to RowData conversion logic in AvroToRowDataConverters.java
+* RowData to Avro conversion logic in RowDataToAvroConverters.java
+* Avro schema conversion logic in AvroSchemaConverter.java
+* Joda optional dependency encapsulation in JodaConverter.java
+* Binary variant accessor utility methods in BinaryVariantAccessorUtils.java
+
+Copyright: 1999-2022 The Apache Software Foundation.
+Project URL: https://flink.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from Delta Lake.
+
+* RoaringPositionBitmap is a Java implementation of RoaringBitmapArray in Delta.
+
+Copyright: 2020 The Delta Lake Project Authors.
+Project URL: https://delta.io/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Commons.
+
+* Core ArrayUtil.
+
+Copyright: 2020 The Apache Software Foundation
+Project URL: https://commons.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles and includes code from Apache HttpComponents (core/client).
+
+* retry and error handling logic in ExponentialHttpRequestRetryStrategy.java
+
+Copyright: 1999-2022 The Apache Software Foundation.
+Project URL: https://hc.apache.org/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles failsafe.
+
+Copyright: Jonathan Halterman and friends
+Project URL: https://failsafe.dev/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles RoaringBitmap.
+
+Copyright: (c) 2013-... the RoaringBitmap authors
+Project URL: https://github.com/RoaringBitmap/RoaringBitmap
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Eclipse MicroProfile OpenAPI.
+
+Copyright: Copyright (c) 2017 Contributors to the Eclipse Foundation
+Project URL: https://github.com/microprofile/microprofile-open-api
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Luben Zstd.
+
+Copyright: Copyright (c) 2015-present, Luben Karavelov/ All rights reserved.
+Project URL: https://github.com/luben/zstd-jni/
+License: BSD 2-Clause
+
+| Zstd-jni: JNI bindings to Zstd Library
+|
+| Copyright (c) 2015-present, Luben Karavelov/ All rights reserved.
+|
+| BSD License
+|
+| Redistribution and use in source and binary forms, with or without modification,
+| are permitted provided that the following conditions are met:
+|
+| * Redistributions of source code must retain the above copyright notice, this
+| list of conditions and the following disclaimer.
+|
+| * Redistributions in binary form must reproduce the above copyright notice, this
+| list of conditions and the following disclaimer in the documentation and/or
+| other materials provided with the distribution.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+| ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+| WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+| DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+| ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+| (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+| LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+| ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This product bundles JTS Topology Suite.
+
+Project URL: https://github.com/locationtech/jts
+License: Eclipse Distribution License v. 1.0 - https://www.eclipse.org/org/documents/edl-v10.php
+
+| Eclipse Distribution License - v 1.0
+|
+| Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.
+|
+| All rights reserved.
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions are met:
+|
+| - Redistributions of source code must retain the above copyright notice,
+| this list of conditions and the following disclaimer.
+|
+| - Redistributions in binary form must reproduce the above copyright notice,
+| this list of conditions and the following disclaimer in the documentation
+| and/or other materials provided with the distribution.
+|
+| - Neither the name of the Eclipse Foundation, Inc. nor the names of its
+| contributors may be used to endorse or promote products derived from this
+| software without specific prior written permission.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+| ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+| CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+| SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+| INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+| CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+| ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+| POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This product bundles the Mozilla Public Suffix List, distributed by Apache HttpComponents.
+
+Project URL: https://publicsuffix.org/
+License: Mozilla Public License, Version 2.0 - https://mozilla.org/MPL/2.0/
+
+| Mozilla Public License Version 2.0
+| ==================================
+|
+| 1. Definitions
+| --------------
+|
+| 1.1. "Contributor"
+| means each individual or legal entity that creates, contributes to
+| the creation of, or owns Covered Software.
+|
+| 1.2. "Contributor Version"
+| means the combination of the Contributions of others (if any) used
+| by a Contributor and that particular Contributor's Contribution.
+|
+| 1.3. "Contribution"
+| means Covered Software of a particular Contributor.
+|
+| 1.4. "Covered Software"
+| means Source Code Form to which the initial Contributor has attached
+| the notice in Exhibit A, the Executable Form of such Source Code
+| Form, and Modifications of such Source Code Form, in each case
+| including portions thereof.
+|
+| 1.5. "Incompatible With Secondary Licenses"
+| means
+|
+| (a) that the initial Contributor has attached the notice described
+| in Exhibit B to the Covered Software; or
+|
+| (b) that the Covered Software was made available under the terms of
+| version 1.1 or earlier of the License, but not also under the
+| terms of a Secondary License.
+|
+| 1.6. "Executable Form"
+| means any form of the work other than Source Code Form.
+|
+| 1.7. "Larger Work"
+| means a work that combines Covered Software with other material, in
+| a separate file or files, that is not Covered Software.
+|
+| 1.8. "License"
+| means this document.
+|
+| 1.9. "Licensable"
+| means having the right to grant, to the maximum extent possible,
+| whether at the time of the initial grant or subsequently, any and
+| all of the rights conveyed by this License.
+|
+| 1.10. "Modifications"
+| means any of the following:
+|
+| (a) any file in Source Code Form that results from an addition to,
+| deletion from, or modification of the contents of Covered
+| Software; or
+|
+| (b) any new file in Source Code Form that contains any Covered
+| Software.
+|
+| 1.11. "Patent Claims" of a Contributor
+| means any patent claim(s), including without limitation, method,
+| process, and apparatus claims, in any patent Licensable by such
+| Contributor that would be infringed, but for the grant of the
+| License, by the making, using, selling, offering for sale, having
+| made, import, or transfer of either its Contributions or its
+| Contributor Version.
+|
+| 1.12. "Secondary License"
+| means either the GNU General Public License, Version 2.0, the GNU
+| Lesser General Public License, Version 2.1, the GNU Affero General
+| Public License, Version 3.0, or any later versions of those
+| licenses.
+|
+| 1.13. "Source Code Form"
+| means the form of the work preferred for making modifications.
+|
+| 1.14. "You" (or "Your")
+| means an individual or a legal entity exercising rights under this
+| License. For legal entities, "You" includes any entity that
+| controls, is controlled by, or is under common control with You. For
+| purposes of this definition, "control" means (a) the power, direct
+| or indirect, to cause the direction or management of such entity,
+| whether by contract or otherwise, or (b) ownership of more than
+| fifty percent (50%) of the outstanding shares or beneficial
+| ownership of such entity.
+|
+| 2. License Grants and Conditions
+| --------------------------------
+|
+| 2.1. Grants
+|
+| Each Contributor hereby grants You a world-wide, royalty-free,
+| non-exclusive license:
+|
+| (a) under intellectual property rights (other than patent or trademark)
+| Licensable by such Contributor to use, reproduce, make available,
+| modify, display, perform, distribute, and otherwise exploit its
+| Contributions, either on an unmodified basis, with Modifications, or
+| as part of a Larger Work; and
+|
+| (b) under Patent Claims of such Contributor to make, use, sell, offer
+| for sale, have made, import, and otherwise transfer either its
+| Contributions or its Contributor Version.
+|
+| 2.2. Effective Date
+|
+| The licenses granted in Section 2.1 with respect to any Contribution
+| become effective for each Contribution on the date the Contributor first
+| distributes such Contribution.
+|
+| 2.3. Limitations on Grant Scope
+|
+| The licenses granted in this Section 2 are the only rights granted under
+| this License. No additional rights or licenses will be implied from the
+| distribution or licensing of Covered Software under this License.
+| Notwithstanding Section 2.1(b) above, no patent license is granted by a
+| Contributor:
+|
+| (a) for any code that a Contributor has removed from Covered Software;
+| or
+|
+| (b) for infringements caused by: (i) Your and any other third party's
+| modifications of Covered Software, or (ii) the combination of its
+| Contributions with other software (except as part of its Contributor
+| Version); or
+|
+| (c) under Patent Claims infringed by Covered Software in the absence of
+| its Contributions.
+|
+| This License does not grant any rights in the trademarks, service marks,
+| or logos of any Contributor (except as may be necessary to comply with
+| the notice requirements in Section 3.4).
+|
+| 2.4. Subsequent Licenses
+|
+| No Contributor makes additional grants as a result of Your choice to
+| distribute the Covered Software under a subsequent version of this
+| License (see Section 10.2) or under the terms of a Secondary License (if
+| permitted under the terms of Section 3.3).
+|
+| 2.5. Representation
+|
+| Each Contributor represents that the Contributor believes its
+| Contributions are its original creation(s) or it has sufficient rights
+| to grant the rights to its Contributions conveyed by this License.
+|
+| 2.6. Fair Use
+|
+| This License is not intended to limit any rights You have under
+| applicable copyright doctrines of fair use, fair dealing, or other
+| equivalents.
+|
+| 2.7. Conditions
+|
+| Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+| in Section 2.1.
+|
+| 3. Responsibilities
+| -------------------
+|
+| 3.1. Distribution of Source Form
+|
+| All distribution of Covered Software in Source Code Form, including any
+| Modifications that You create or to which You contribute, must be under
+| the terms of this License. You must inform recipients that the Source
+| Code Form of the Covered Software is governed by the terms of this
+| License, and how they can obtain a copy of this License. You may not
+| attempt to alter or restrict the recipients' rights in the Source Code
+| Form.
+|
+| 3.2. Distribution of Executable Form
+|
+| If You distribute Covered Software in Executable Form then:
+|
+| (a) such Covered Software must also be made available in Source Code
+| Form, as described in Section 3.1, and You must inform recipients of
+| the Executable Form how they can obtain a copy of such Source Code
+| Form by reasonable means in a timely manner, at a charge no more
+| than the cost of distribution to the recipient; and
+|
+| (b) You may distribute such Executable Form under the terms of this
+| License, or sublicense it under different terms, provided that the
+| license for the Executable Form does not attempt to limit or alter
+| the recipients' rights in the Source Code Form under this License.
+|
+| 3.3. Distribution of a Larger Work
+|
+| You may create and distribute a Larger Work under terms of Your choice,
+| provided that You also comply with the requirements of this License for
+| the Covered Software. If the Larger Work is a combination of Covered
+| Software with a work governed by one or more Secondary Licenses, and the
+| Covered Software is not Incompatible With Secondary Licenses, this
+| License permits You to additionally distribute such Covered Software
+| under the terms of such Secondary License(s), so that the recipient of
+| the Larger Work may, at their option, further distribute the Covered
+| Software under the terms of either this License or such Secondary
+| License(s).
+|
+| 3.4. Notices
+|
+| You may not remove or alter the substance of any license notices
+| (including copyright notices, patent notices, disclaimers of warranty,
+| or limitations of liability) contained within the Source Code Form of
+| the Covered Software, except that You may alter any license notices to
+| the extent required to remedy known factual inaccuracies.
+|
+| 3.5. Application of Additional Terms
+|
+| You may choose to offer, and to charge a fee for, warranty, support,
+| indemnity or liability obligations to one or more recipients of Covered
+| Software. However, You may do so only on Your own behalf, and not on
+| behalf of any Contributor. You must make it absolutely clear that any
+| such warranty, support, indemnity, or liability obligation is offered by
+| You alone, and You hereby agree to indemnify every Contributor for any
+| liability incurred by such Contributor as a result of warranty, support,
+| indemnity or liability terms You offer. You may include additional
+| disclaimers of warranty and limitations of liability specific to any
+| jurisdiction.
+|
+| 4. Inability to Comply Due to Statute or Regulation
+| ---------------------------------------------------
+|
+| If it is impossible for You to comply with any of the terms of this
+| License with respect to some or all of the Covered Software due to
+| statute, judicial order, or regulation then You must: (a) comply with
+| the terms of this License to the maximum extent possible; and (b)
+| describe the limitations and the code they affect. Such description must
+| be placed in a text file included with all distributions of the Covered
+| Software under this License. Except to the extent prohibited by statute
+| or regulation, such description must be sufficiently detailed for a
+| recipient of ordinary skill to be able to understand it.
+|
+| 5. Termination
+| --------------
+|
+| 5.1. The rights granted under this License will terminate automatically
+| if You fail to comply with any of its terms. However, if You become
+| compliant, then the rights granted under this License from a particular
+| Contributor are reinstated (a) provisionally, unless and until such
+| Contributor explicitly and finally terminates Your grants, and (b) on an
+| ongoing basis, if such Contributor fails to notify You of the
+| non-compliance by some reasonable means prior to 60 days after You have
+| come back into compliance. Moreover, Your grants from a particular
+| Contributor are reinstated on an ongoing basis if such Contributor
+| notifies You of the non-compliance by some reasonable means, this is the
+| first time You have received notice of non-compliance with this License
+| from such Contributor, and You become compliant prior to 30 days after
+| Your receipt of the notice.
+|
+| 5.2. If You initiate litigation against any entity by asserting a patent
+| infringement claim (excluding declaratory judgment actions,
+| counter-claims, and cross-claims) alleging that a Contributor Version
+| directly or indirectly infringes any patent, then the rights granted to
+| You by any and all Contributors for the Covered Software under Section
+| 2.1 of this License shall terminate.
+|
+| 5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+| end user license agreements (excluding distributors and resellers) which
+| have been validly granted by You or Your distributors under this License
+| prior to termination shall survive termination.
+|
+| ************************************************************************
+| * *
+| * 6. Disclaimer of Warranty *
+| * ------------------------- *
+| * *
+| * Covered Software is provided under this License on an "as is" *
+| * basis, without warranty of any kind, either expressed, implied, or *
+| * statutory, including, without limitation, warranties that the *
+| * Covered Software is free of defects, merchantable, fit for a *
+| * particular purpose or non-infringing. The entire risk as to the *
+| * quality and performance of the Covered Software is with You. *
+| * Should any Covered Software prove defective in any respect, You *
+| * (not any Contributor) assume the cost of any necessary servicing, *
+| * repair, or correction. This disclaimer of warranty constitutes an *
+| * essential part of this License. No use of any Covered Software is *
+| * authorized under this License except under this disclaimer. *
+| * *
+| ************************************************************************
+|
+| ************************************************************************
+| * *
+| * 7. Limitation of Liability *
+| * -------------------------- *
+| * *
+| * Under no circumstances and under no legal theory, whether tort *
+| * (including negligence), contract, or otherwise, shall any *
+| * Contributor, or anyone who distributes Covered Software as *
+| * permitted above, be liable to You for any direct, indirect, *
+| * special, incidental, or consequential damages of any character *
+| * including, without limitation, damages for lost profits, loss of *
+| * goodwill, work stoppage, computer failure or malfunction, or any *
+| * and all other commercial damages or losses, even if such party *
+| * shall have been informed of the possibility of such damages. This *
+| * limitation of liability shall not apply to liability for death or *
+| * personal injury resulting from such party's negligence to the *
+| * extent applicable law prohibits such limitation. Some *
+| * jurisdictions do not allow the exclusion or limitation of *
+| * incidental or consequential damages, so this exclusion and *
+| * limitation may not apply to You. *
+| * *
+| ************************************************************************
+|
+| 8. Litigation
+| -------------
+|
+| Any litigation relating to this License may be brought only in the
+| courts of a jurisdiction where the defendant maintains its principal
+| place of business and such litigation shall be governed by laws of that
+| jurisdiction, without reference to its conflict-of-law provisions.
+| Nothing in this Section shall prevent a party's ability to bring
+| cross-claims or counter-claims.
+|
+| 9. Miscellaneous
+| ----------------
+|
+| This License represents the complete agreement concerning the subject
+| matter hereof. If any provision of this License is held to be
+| unenforceable, such provision shall be reformed only to the extent
+| necessary to make it enforceable. Any law or regulation which provides
+| that the language of a contract shall be construed against the drafter
+| shall not be used to construe this License against a Contributor.
+|
+| 10. Versions of the License
+| ---------------------------
+|
+| 10.1. New Versions
+|
+| Mozilla Foundation is the license steward. Except as provided in Section
+| 10.3, no one other than the license steward has the right to modify or
+| publish new versions of this License. Each version will be given a
+| distinguishing version number.
+|
+| 10.2. Effect of New Versions
+|
+| You may distribute the Covered Software under the terms of the version
+| of the License under which You originally received the Covered Software,
+| or under the terms of any subsequent version published by the license
+| steward.
+|
+| 10.3. Modified Versions
+|
+| If you create software not governed by this License, and you want to
+| create a new license for such software, you may create and use a
+| modified version of this License if you rename the license and remove
+| any references to the name of the license steward (except to note that
+| such modified license differs from this License).
+|
+| 10.4. Distributing Source Code Form that is Incompatible With Secondary
+| Licenses
+|
+| If You choose to distribute Source Code Form that is Incompatible With
+| Secondary Licenses under the terms of this version of the License, the
+| notice described in Exhibit B of this License must be attached.
+|
+| Exhibit A - Source Code Form License Notice
+| -------------------------------------------
+|
+| This Source Code Form is subject to the terms of the Mozilla Public
+| License, v. 2.0. If a copy of the MPL was not distributed with this
+| file, You can obtain one at http://mozilla.org/MPL/2.0/.
+|
+| If it is not possible or desirable to put the notice in a particular
+| file, then You may include the notice in a location (such as a LICENSE
+| file in a relevant directory) where a recipient would be likely to look
+| for such a notice.
+|
+| You may add additional accurate notices of copyright ownership.
+|
+| Exhibit B - "Incompatible With Secondary Licenses" Notice
+| ---------------------------------------------------------
+|
+| This Source Code Form is "Incompatible With Secondary Licenses", as
+| defined by the Mozilla Public License, v. 2.0.
+
+--------------------------------------------------------------------------------
+
diff --git a/flink/v2.2/flink-runtime/NOTICE b/flink/v2.2/flink-runtime/NOTICE
new file mode 100644
index 0000000000..9887b467aa
--- /dev/null
+++ b/flink/v2.2/flink-runtime/NOTICE
@@ -0,0 +1,439 @@
+
+Apache Iceberg
+Copyright 2017-2026 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+--------------------------------------------------------------------------------
+
+This product includes code from Kite, developed at Cloudera, Inc. with
+the following in its NOTICE file:
+| Copyright 2013 Cloudera Inc.
+|
+| Licensed under the Apache License, Version 2.0 (the "License");
+| you may not use this file except in compliance with the License.
+| You may obtain a copy of the License at
+|
+| http://www.apache.org/licenses/LICENSE-2.0
+|
+| Unless required by applicable law or agreed to in writing, software
+| distributed under the License is distributed on an "AS IS" BASIS,
+| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+| See the License for the specific language governing permissions and
+| limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Presto, developed at Facebook, with the
+following copyright notice:
+
+| Copyright 2016 Facebook and contributors.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Delta Lake, developed at The Delta Lake Project,
+with the following copyright notice:
+
+| Copyright (2021) The Delta Lake Project Authors.
+
+--------------------------------------------------------------------------------
+
+This product bundles Airlift Aircompressor with the following in its NOTICE file:
+| Snappy Copyright Notices
+| =========================
+|
+| * Copyright 2011 Dain Sundstrom <da...@iq80.com>
+| * Copyright 2011, Google Inc.<op...@google.com>
+|
+|
+| Snappy License
+| ===============
+| Copyright 2011, Google Inc.
+| All rights reserved.
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions are
+| met:
+|
+| * Redistributions of source code must retain the above copyright
+| notice, this list of conditions and the following disclaimer.
+| * Redistributions in binary form must reproduce the above
+| copyright notice, this list of conditions and the following disclaimer
+| in the documentation and/or other materials provided with the
+| distribution.
+| * Neither the name of Google Inc. nor the names of its
+| contributors may be used to endorse or promote products derived from
+| this software without specific prior written permission.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This product bundles Project Nessie with the following in its NOTICE file:
+| Nessie
+| Copyright 2015-2025 Dremio Corporation
+|
+| ---------------------------------------
+| This project includes code from Apache Polaris (incubating), with the following in its NOTICE file:
+|
+| | Apache Polaris (incubating)
+| | Copyright 2024 The Apache Software Foundation
+| |
+| | This product includes software developed at
+| | The Apache Software Foundation (http://www.apache.org/).
+| |
+| | The initial code for the Polaris project was donated
+| | to the ASF by Snowflake Inc. (https://www.snowflake.com/) copyright 2024.
+|
+| ---------------------------------------
+| This project includes code from Netty, with the following in its NOTICE file:
+|
+| | The Netty Project
+| | =================
+| |
+| | Please visit the Netty web site for more information:
+| |
+| | * https://netty.io/
+| |
+| | Copyright 2014 The Netty Project
+| |
+| | The Netty Project licenses this file to you under the Apache License,
+| | version 2.0 (the "License"); you may not use this file except in compliance
+| | with the License. You may obtain a copy of the License at:
+| |
+| | https://www.apache.org/licenses/LICENSE-2.0
+| |
+| | Unless required by applicable law or agreed to in writing, software
+| | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+| | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+| | License for the specific language governing permissions and limitations
+| | under the License.
+| |
+| | Also, please refer to each LICENSE.<component>.txt file, which is located in
+| | the 'license' directory of the distribution file, for the license terms of the
+| | components that this product depends on.
+| |
+| | -------------------------------------------------------------------------------
+| | This product contains the extensions to Java Collections Framework which has
+| | been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
+| |
+| | * LICENSE:
+| | * license/LICENSE.jsr166y.txt (Public Domain)
+| | * HOMEPAGE:
+| | * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
+| | * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
+| |
+| | This product contains a modified version of Robert Harder's Public Domain
+| | Base64 Encoder and Decoder, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.base64.txt (Public Domain)
+| | * HOMEPAGE:
+| | * http://iharder.sourceforge.net/current/java/base64/
+| |
+| | This product contains a modified portion of 'Webbit', an event based
+| | WebSocket and HTTP server, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.webbit.txt (BSD License)
+| | * HOMEPAGE:
+| | * https://github.com/joewalnes/webbit
+| |
+| | This product contains a modified portion of 'SLF4J', a simple logging
+| | facade for Java, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.slf4j.txt (MIT License)
+| | * HOMEPAGE:
+| | * https://www.slf4j.org/
+| |
+| | This product contains a modified portion of 'Apache Harmony', an open source
+| | Java SE, which can be obtained at:
+| |
+| | * NOTICE:
+| | * license/NOTICE.harmony.txt
+| | * LICENSE:
+| | * license/LICENSE.harmony.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://archive.apache.org/dist/harmony/
+| |
+| | This product contains a modified portion of 'jbzip2', a Java bzip2 compression
+| | and decompression library written by Matthew J. Francis. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.jbzip2.txt (MIT License)
+| | * HOMEPAGE:
+| | * https://code.google.com/p/jbzip2/
+| |
+| | This product contains a modified portion of 'libdivsufsort', a C API library to construct
+| | the suffix array and the Burrows-Wheeler transformed string for any input string of
+| | a constant-size alphabet written by Yuta Mori. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.libdivsufsort.txt (MIT License)
+| | * HOMEPAGE:
+| | * https://github.com/y-256/libdivsufsort
+| |
+| | This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM,
+| | which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.jctools.txt (ASL2 License)
+| | * HOMEPAGE:
+| | * https://github.com/JCTools/JCTools
+| |
+| | This product optionally depends on 'JZlib', a re-implementation of zlib in
+| | pure Java, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.jzlib.txt (BSD style License)
+| | * HOMEPAGE:
+| | * http://www.jcraft.com/jzlib/
+| |
+| | This product optionally depends on 'Compress-LZF', a Java library for encoding and
+| | decoding data in LZF format, written by Tatu Saloranta. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.compress-lzf.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/ning/compress
+| |
+| | This product optionally depends on 'lz4', a LZ4 Java compression
+| | and decompression library written by Adrien Grand. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.lz4.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/jpountz/lz4-java
+| |
+| | This product optionally depends on 'lzma-java', a LZMA Java compression
+| | and decompression library, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.lzma-java.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/jponge/lzma-java
+| |
+| | This product optionally depends on 'zstd-jni', a zstd-jni Java compression
+| | and decompression library, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.zstd-jni.txt (BSD)
+| | * HOMEPAGE:
+| | * https://github.com/luben/zstd-jni
+| |
+| | This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression
+| | and decompression library written by William Kinney. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.jfastlz.txt (MIT License)
+| | * HOMEPAGE:
+| | * https://code.google.com/p/jfastlz/
+| |
+| | This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data
+| | interchange format, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.protobuf.txt (New BSD License)
+| | * HOMEPAGE:
+| | * https://github.com/google/protobuf
+| |
+| | This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
+| | a temporary self-signed X.509 certificate when the JVM does not provide the
+| | equivalent functionality. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.bouncycastle.txt (MIT License)
+| | * HOMEPAGE:
+| | * https://www.bouncycastle.org/
+| |
+| | This product optionally depends on 'Snappy', a compression library produced
+| | by Google Inc, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.snappy.txt (New BSD License)
+| | * HOMEPAGE:
+| | * https://github.com/google/snappy
+| |
+| | This product optionally depends on 'JBoss Marshalling', an alternative Java
+| | serialization API, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.jboss-marshalling.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/jboss-remoting/jboss-marshalling
+| |
+| | This product optionally depends on 'Caliper', Google's micro-
+| | benchmarking framework, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.caliper.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/google/caliper
+| |
+| | This product optionally depends on 'Apache Commons Logging', a logging
+| | framework, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.commons-logging.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://commons.apache.org/logging/
+| |
+| | This product optionally depends on 'Apache Log4J', a logging framework, which
+| | can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.log4j.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://logging.apache.org/log4j/
+| |
+| | This product optionally depends on 'Aalto XML', an ultra-high performance
+| | non-blocking XML processor, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.aalto-xml.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://wiki.fasterxml.com/AaltoHome
+| |
+| | This product contains a modified version of 'HPACK', a Java implementation of
+| | the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.hpack.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/twitter/hpack
+| |
+| | This product contains a modified version of 'HPACK', a Java implementation of
+| | the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.hyper-hpack.txt (MIT License)
+| | * HOMEPAGE:
+| | * https://github.com/python-hyper/hpack/
+| |
+| | This product contains a modified version of 'HPACK', a Java implementation of
+| | the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.nghttp2-hpack.txt (MIT License)
+| | * HOMEPAGE:
+| | * https://github.com/nghttp2/nghttp2/
+| |
+| | This product contains a modified portion of 'Apache Commons Lang', a Java library
+| | provides utilities for the java.lang API, which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.commons-lang.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://commons.apache.org/proper/commons-lang/
+| |
+| |
+| | This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build.
+| |
+| | * LICENSE:
+| | * license/LICENSE.mvn-wrapper.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/takari/maven-wrapper
+| |
+| | This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS.
+| | This private header is also used by Apple's open source
+| | mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/).
+| |
+| | * LICENSE:
+| | * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0)
+| | * HOMEPAGE:
+| | * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
+| |
+| | This product optionally depends on 'Brotli4j', Brotli compression and
+| | decompression for Java., which can be obtained at:
+| |
+| | * LICENSE:
+| | * license/LICENSE.brotli4j.txt (Apache License 2.0)
+| | * HOMEPAGE:
+| | * https://github.com/hyperxpro/Brotli4j
+
+--------------------------------------------------------------------------------
+
+This product bundles Eclipse Microprofile OpenAPI with the following in its NOTICE file:
+| =========================================================================
+| == NOTICE file corresponding to section 4(d) of the Apache License, ==
+| == Version 2.0, in this case for MicroProfile OpenAPI ==
+| =========================================================================
+|
+| The majority of this software were originally based on the following:
+| * Swagger Core
+| https://github.com/swagger-api/swagger-core
+| under Apache License, v2.0
+|
+|
+| SPDXVersion: SPDX-2.1
+| PackageName: Eclipse MicroProfile
+| PackageHomePage: http://www.eclipse.org/microprofile
+| PackageLicenseDeclared: Apache-2.0
+|
+| PackageCopyrightText: <text>
+| Arthur De Magalhaes arthurdm@ca.ibm.com
+| </text>
+
+--------------------------------------------------------------------------------
+
+This product bundles Jackson JSON Processor with the following in its NOTICE file:
+| # Jackson JSON processor
+|
+| Jackson is a high-performance, Free/Open Source JSON processing library.
+| It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
+| been in development since 2007.
+| It is currently developed by a community of developers.
+|
+| ## Copyright
+|
+| Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi)
+|
+| ## Licensing
+|
+| Jackson 2.x core and extension components are licensed under Apache License 2.0
+| To find the details that apply to this artifact see the accompanying LICENSE file.
+|
+| ## Credits
+|
+| A list of contributors may be found from CREDITS(-2.x) file, which is included
+| in some artifacts (usually source distributions); but is always available
+| from the source code management (SCM) system project uses.
+|
+| ## FastDoubleParser
+|
+| jackson-core bundles a shaded copy of FastDoubleParser <https://github.com/wrandelshofer/FastDoubleParser>.
+| That code is available under an MIT license <https://github.com/wrandelshofer/FastDoubleParser/blob/main/LICENSE>
+| under the following copyright.
+|
+| Copyright © 2023 Werner Randelshofer, Switzerland. MIT License.
+|
+| See FastDoubleParser-NOTICE for details of other source code included in FastDoubleParser
+| and the licenses and copyrights that apply to that code.
+
+--------------------------------------------------------------------------------
+
+This product bundles Apache DataSketches Memory with the following in its NOTICE file:
+| Apache DataSketches Memory
+| Copyright 2024 - The Apache Software Foundation
+|
+| Copyright 2015-2018 Yahoo Inc.
+| Copyright 2019-2020 Verizon Media
+| Copyright 2021 Yahoo Inc.
+|
+| This product includes software developed at
+| The Apache Software Foundation (http://www.apache.org/).
+|
+| Prior to moving to ASF, the software for this project was developed at
+| Yahoo Inc. (https://developer.yahoo.com).
diff --git a/flink/v2.2/flink-runtime/runtime-deps.txt b/flink/v2.2/flink-runtime/runtime-deps.txt
new file mode 100644
index 0000000000..224e19d09f
--- /dev/null
+++ b/flink/v2.2/flink-runtime/runtime-deps.txt
@@ -0,0 +1,31 @@
+com.fasterxml.jackson.core:jackson-annotations:2.22
+com.fasterxml.jackson.core:jackson-core:2.22
+com.fasterxml.jackson.core:jackson-databind:2.22
+com.github.ben-manes.caffeine:caffeine:3.2
+com.github.luben:zstd-jni:1.5
+com.google.errorprone:error_prone_annotations:2.49
+dev.failsafe:failsafe:3.3
+io.airlift:aircompressor:2.0
+org.apache.avro:avro:1.12
+org.apache.datasketches:datasketches-java:6.2
+org.apache.datasketches:datasketches-memory:3.0
+org.apache.httpcomponents.client5:httpclient5:5.6
+org.apache.httpcomponents.core5:httpcore5-h2:5.4
+org.apache.httpcomponents.core5:httpcore5:5.4
+org.apache.orc:orc-core:1.9
+org.apache.orc:orc-shims:1.9
+org.apache.parquet:parquet-avro:1.17
+org.apache.parquet:parquet-column:1.17
+org.apache.parquet:parquet-common:1.17
+org.apache.parquet:parquet-encoding:1.17
+org.apache.parquet:parquet-format-structures:1.17
+org.apache.parquet:parquet-hadoop:1.17
+org.apache.parquet:parquet-jackson:1.17
+org.apache.parquet:parquet-variant:1.17
+org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1
+org.jspecify:jspecify:1.0
+org.locationtech.jts:jts-core:1.20
+org.projectnessie.nessie:nessie-client:0.108
+org.projectnessie.nessie:nessie-model:0.108
+org.roaringbitmap:RoaringBitmap:1.6
+org.threeten:threeten-extra:1.7
diff --git a/flink/v2.2/flink-runtime/src/integration/java/org/apache/iceberg/flink/TestIcebergConnectorSmoke.java b/flink/v2.2/flink-runtime/src/integration/java/org/apache/iceberg/flink/TestIcebergConnectorSmoke.java
new file mode 100644
index 0000000000..3ba3bb71a1
--- /dev/null
+++ b/flink/v2.2/flink-runtime/src/integration/java/org/apache/iceberg/flink/TestIcebergConnectorSmoke.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+public class TestIcebergConnectorSmoke extends TestIcebergConnector {}
diff --git a/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/dynamic/DynamicRecordSerializerDeserializerBenchmark.java b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/dynamic/DynamicRecordSerializerDeserializerBenchmark.java
new file mode 100644
index 0000000000..6e5d04499e
--- /dev/null
+++ b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/dynamic/DynamicRecordSerializerDeserializerBenchmark.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.sink.dynamic;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.flink.api.common.serialization.SerializerConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputSerializer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.data.RandomGenericData;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.flink.CatalogLoader;
+import org.apache.iceberg.flink.RowDataConverter;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.SingleShotTime)
+public class DynamicRecordSerializerDeserializerBenchmark {
+ private static final int SAMPLE_SIZE = 100_000;
+ private static final Schema SCHEMA =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.required(2, "name2", Types.StringType.get()),
+ Types.NestedField.required(3, "name3", Types.StringType.get()),
+ Types.NestedField.required(4, "name4", Types.StringType.get()),
+ Types.NestedField.required(5, "name5", Types.StringType.get()),
+ Types.NestedField.required(6, "name6", Types.StringType.get()),
+ Types.NestedField.required(7, "name7", Types.StringType.get()),
+ Types.NestedField.required(8, "name8", Types.StringType.get()),
+ Types.NestedField.required(9, "name9", Types.StringType.get()));
+
+ private List<DynamicRecordInternal> rows = Lists.newArrayListWithExpectedSize(SAMPLE_SIZE);
+ private DynamicRecordInternalType type;
+
+ public static void main(String[] args) throws RunnerException {
+ Options options =
+ new OptionsBuilder()
+ .include(DynamicRecordSerializerDeserializerBenchmark.class.getSimpleName())
+ .build();
+ new Runner(options).run();
+ }
+
+ @Setup
+ public void setupBenchmark() throws IOException {
+ List<Record> records = RandomGenericData.generate(SCHEMA, SAMPLE_SIZE, 1L);
+ this.rows =
+ records.stream()
+ .map(
+ r ->
+ new DynamicRecordInternal(
+ "t",
+ SnapshotRef.MAIN_BRANCH,
+ SCHEMA,
+ RowDataConverter.convert(SCHEMA, r),
+ PartitionSpec.unpartitioned(),
+ 1,
+ false,
+ Collections.emptySet()))
+ .collect(Collectors.toList());
+
+ File warehouse = Files.createTempFile("perf-bench", null).toFile();
+ CatalogLoader catalogLoader =
+ CatalogLoader.hadoop(
+ "hadoop",
+ new Configuration(),
+ ImmutableMap.of(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getPath()));
+ this.type = new DynamicRecordInternalType(catalogLoader, true, 100);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void testSerialize(Blackhole blackhole) throws IOException {
+ TypeSerializer<DynamicRecordInternal> serializer =
+ type.createSerializer((SerializerConfig) null);
+ DataOutputSerializer outputView = new DataOutputSerializer(1024);
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ serializer.serialize(rows.get(i), outputView);
+ }
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void testSerializeAndDeserialize(Blackhole blackhole) throws IOException {
+ TypeSerializer<DynamicRecordInternal> serializer =
+ type.createSerializer((SerializerConfig) null);
+
+ DataOutputSerializer outputView = new DataOutputSerializer(1024);
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ serializer.serialize(rows.get(i), outputView);
+ serializer.deserialize(new DataInputDeserializer(outputView.getSharedBuffer()));
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java
new file mode 100644
index 0000000000..80a46ac530
--- /dev/null
+++ b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.sink.shuffle;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortKey;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.SortOrderComparators;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.SingleShotTime)
+public class MapRangePartitionerBenchmark {
+
+ private static final int SAMPLE_SIZE = 100_000;
+ private static final Schema SCHEMA =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.required(2, "name2", Types.StringType.get()),
+ Types.NestedField.required(3, "name3", Types.StringType.get()),
+ Types.NestedField.required(4, "name4", Types.StringType.get()),
+ Types.NestedField.required(5, "name5", Types.StringType.get()),
+ Types.NestedField.required(6, "name6", Types.StringType.get()),
+ Types.NestedField.required(7, "name7", Types.StringType.get()),
+ Types.NestedField.required(8, "name8", Types.StringType.get()),
+ Types.NestedField.required(9, "name9", Types.StringType.get()));
+
+ private static final SortOrder SORT_ORDER = SortOrder.builderFor(SCHEMA).asc("id").build();
+ private static final Comparator<StructLike> SORT_ORDER_COMPARTOR =
+ SortOrderComparators.forSchema(SCHEMA, SORT_ORDER);
+ private static final SortKey SORT_KEY = new SortKey(SCHEMA, SORT_ORDER);
+ private static final int PARALLELISM = 100;
+
+ private MapRangePartitioner partitioner;
+ private RowData[] rows;
+
+ @Setup
+ public void setupBenchmark() {
+ NavigableMap<Integer, Long> weights =
+ DataDistributionUtil.longTailDistribution(100_000, 24, 240, 100, 2.0, 0.7);
+ Map<SortKey, Long> mapStatistics =
+ DataDistributionUtil.mapStatisticsWithLongTailDistribution(weights, SORT_KEY);
+
+ MapAssignment mapAssignment =
+ MapAssignment.fromKeyFrequency(PARALLELISM, mapStatistics, 0.0, SORT_ORDER_COMPARTOR);
+ this.partitioner = new MapRangePartitioner(SCHEMA, SORT_ORDER, mapAssignment);
+
+ List<Integer> keys = Lists.newArrayList(weights.keySet().iterator());
+ long[] weightsCDF = DataDistributionUtil.computeCumulativeWeights(keys, weights);
+ long totalWeight = weightsCDF[weightsCDF.length - 1];
+
+ // pre-calculate the samples for benchmark run
+ this.rows = new GenericRowData[SAMPLE_SIZE];
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ long weight = ThreadLocalRandom.current().nextLong(totalWeight);
+ int index = DataDistributionUtil.binarySearchIndex(weightsCDF, weight);
+ rows[i] =
+ GenericRowData.of(
+ keys.get(index),
+ DataDistributionUtil.randomString("name2-", 200),
+ DataDistributionUtil.randomString("name3-", 200),
+ DataDistributionUtil.randomString("name4-", 200),
+ DataDistributionUtil.randomString("name5-", 200),
+ DataDistributionUtil.randomString("name6-", 200),
+ DataDistributionUtil.randomString("name7-", 200),
+ DataDistributionUtil.randomString("name8-", 200),
+ DataDistributionUtil.randomString("name9-", 200));
+ }
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {}
+
+ @Benchmark
+ @Threads(1)
+ public void testPartitionerLongTailDistribution(Blackhole blackhole) {
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ blackhole.consume(partitioner.partition(rows[i], PARALLELISM));
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/SketchRangePartitionerBenchmark.java b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/SketchRangePartitionerBenchmark.java
new file mode 100644
index 0000000000..53a24cd896
--- /dev/null
+++ b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/SketchRangePartitionerBenchmark.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.sink.shuffle;
+
+import java.util.Arrays;
+import java.util.UUID;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortKey;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.types.Types;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.SingleShotTime)
+public class SketchRangePartitionerBenchmark {
+
+ private static final int SAMPLE_SIZE = 100_000;
+ private static final Schema SCHEMA =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.UUIDType.get()),
+ Types.NestedField.required(2, "name2", Types.StringType.get()),
+ Types.NestedField.required(3, "name3", Types.StringType.get()),
+ Types.NestedField.required(4, "name4", Types.StringType.get()),
+ Types.NestedField.required(5, "name5", Types.StringType.get()),
+ Types.NestedField.required(6, "name6", Types.StringType.get()),
+ Types.NestedField.required(7, "name7", Types.StringType.get()),
+ Types.NestedField.required(8, "name8", Types.StringType.get()),
+ Types.NestedField.required(9, "name9", Types.StringType.get()));
+
+ private static final SortOrder SORT_ORDER = SortOrder.builderFor(SCHEMA).asc("id").build();
+ private static final SortKey SORT_KEY = new SortKey(SCHEMA, SORT_ORDER);
+ private static final int PARALLELISM = 100;
+
+ private SketchRangePartitioner partitioner;
+ private RowData[] rows;
+
+ @Setup
+ public void setupBenchmark() {
+ UUID[] reservoir = DataDistributionUtil.reservoirSampleUUIDs(1_000_000, 100_000);
+ UUID[] rangeBound = DataDistributionUtil.rangeBoundSampleUUIDs(reservoir, PARALLELISM);
+ SortKey[] rangeBoundSortKeys =
+ Arrays.stream(rangeBound)
+ .map(
+ uuid -> {
+ SortKey sortKeyCopy = SORT_KEY.copy();
+ sortKeyCopy.set(0, uuid);
+ return sortKeyCopy;
+ })
+ .toArray(SortKey[]::new);
+
+ this.partitioner = new SketchRangePartitioner(SCHEMA, SORT_ORDER, rangeBoundSortKeys);
+
+ // pre-calculate the samples for benchmark run
+ this.rows = new GenericRowData[SAMPLE_SIZE];
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ UUID uuid = UUID.randomUUID();
+ Object uuidBytes = DataDistributionUtil.uuidBytes(uuid);
+ rows[i] =
+ GenericRowData.of(
+ uuidBytes,
+ DataDistributionUtil.randomString("name2-", 200),
+ DataDistributionUtil.randomString("name3-", 200),
+ DataDistributionUtil.randomString("name4-", 200),
+ DataDistributionUtil.randomString("name5-", 200),
+ DataDistributionUtil.randomString("name6-", 200),
+ DataDistributionUtil.randomString("name7-", 200),
+ DataDistributionUtil.randomString("name8-", 200),
+ DataDistributionUtil.randomString("name9-", 200));
+ }
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {}
+
+ @Benchmark
+ @Threads(1)
+ public void testPartitionerLongTailDistribution(Blackhole blackhole) {
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ blackhole.consume(partitioner.partition(rows[i], PARALLELISM));
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/StatisticsRecordSerializerBenchmark.java b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/StatisticsRecordSerializerBenchmark.java
new file mode 100644
index 0000000000..c83c4ada09
--- /dev/null
+++ b/flink/v2.2/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/StatisticsRecordSerializerBenchmark.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.sink.shuffle;
+
+import java.io.IOException;
+import java.util.Random;
+import org.apache.flink.api.common.serialization.SerializerConfigImpl;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputSerializer;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.flink.FlinkSchemaUtil;
+import org.apache.iceberg.types.Types;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.SingleShotTime)
+public class StatisticsRecordSerializerBenchmark {
+ private static final int SAMPLE_SIZE = 100_000;
+ private static final Schema SCHEMA =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.required(2, "name2", Types.StringType.get()),
+ Types.NestedField.required(3, "name3", Types.StringType.get()),
+ Types.NestedField.required(4, "name4", Types.StringType.get()),
+ Types.NestedField.required(5, "name5", Types.StringType.get()),
+ Types.NestedField.required(6, "name6", Types.StringType.get()),
+ Types.NestedField.required(7, "name7", Types.StringType.get()),
+ Types.NestedField.required(8, "name8", Types.StringType.get()),
+ Types.NestedField.required(9, "name9", Types.StringType.get()),
+ Types.NestedField.required(10, "name10", Types.StringType.get()));
+
+ private static final SortOrder SORT_ORDER = SortOrder.builderFor(SCHEMA).asc("id").build();
+ private static final StatisticsOrRecordTypeInformation CUSTOM_TYPE_INFORMATION =
+ new StatisticsOrRecordTypeInformation(FlinkSchemaUtil.convert(SCHEMA), SCHEMA, SORT_ORDER);
+ private static final TypeSerializer<StatisticsOrRecord> CUSTOM_SERIALIZER =
+ CUSTOM_TYPE_INFORMATION.createSerializer(new SerializerConfigImpl());
+ private static final TypeInformation<StatisticsOrRecord> DEFAULT_TYPE_INFORMATION =
+ TypeInformation.of(StatisticsOrRecord.class);
+ private static final TypeSerializer<StatisticsOrRecord> DEFAULT_SERIALIZER =
+ DEFAULT_TYPE_INFORMATION.createSerializer(new SerializerConfigImpl());
+
+ private StatisticsOrRecord[] rows;
+
+ @Setup
+ public void setupBenchmark() {
+ Random random = new Random();
+ rows = new StatisticsOrRecord[SAMPLE_SIZE];
+ for (int i = 0; i < SAMPLE_SIZE; i++) {
+ RowData rowData =
+ GenericRowData.of(
+ random.nextInt(),
+ StringData.fromString(DataDistributionUtil.randomString("name2-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name3-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name4-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name5-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name6-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name7-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name8-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name9-", 200)),
+ StringData.fromString(DataDistributionUtil.randomString("name10-", 200)));
+ rows[i] = StatisticsOrRecord.fromRecord(rowData);
+ }
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {}
+
+ @Benchmark
+ @Threads(1)
+ public void testCustomSerializer(Blackhole blackhole) throws Exception {
+ testSerializer(CUSTOM_SERIALIZER, blackhole);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void testDefaultSerializer(Blackhole blackhole) throws Exception {
+ testSerializer(DEFAULT_SERIALIZER, blackhole);
+ }
+
+ private void testSerializer(TypeSerializer<StatisticsOrRecord> serializer, Blackhole blackhole)
+ throws IOException {
+ DataOutputSerializer outputView = new DataOutputSerializer(1024 * 4);
+ for (int i = 0; i < SAMPLE_SIZE; ++i) {
+ outputView.clear();
+ serializer.serialize(rows[i], outputView);
+ StatisticsOrRecord deserialized =
+ serializer.deserialize(new DataInputDeserializer(outputView.getSharedBuffer()));
+ blackhole.consume(deserialized);
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/flink/table/api/runtime/types/FlinkScalaKryoInstantiator.java b/flink/v2.2/flink/src/main/java/org/apache/flink/table/api/runtime/types/FlinkScalaKryoInstantiator.java
new file mode 100644
index 0000000000..7318c147a1
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/flink/table/api/runtime/types/FlinkScalaKryoInstantiator.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.table.api.runtime.types;
+
+/**
+ * Override Flink's internal FlinkScalaKryoInstantiator to avoid loading the Scala extensions for
+ * the KryoSerializer. This is a workaround until Kryo-related issues with the Scala extensions are
+ * fixed. See: https://issues.apache.org/jira/browse/FLINK-37546
+ */
+public class FlinkScalaKryoInstantiator {}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/flink/types/variant/BinaryVariantAccessorUtils.java b/flink/v2.2/flink/src/main/java/org/apache/flink/types/variant/BinaryVariantAccessorUtils.java
new file mode 100644
index 0000000000..f61cd76074
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/flink/types/variant/BinaryVariantAccessorUtils.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.types.variant;
+
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+
+/**
+ * These Accessor utils should be part of Flink Variant interface itself. Added in the PR, <a
+ * href="https://github.com/apache/flink/pull/27600">...</a>. This is a workaround until we upgrade
+ * Flink to a release version with above changes.
+ */
+public class BinaryVariantAccessorUtils extends BinaryVariantUtil {
+ /**
+ * Get the size of an array variant.
+ *
+ * @return Number of elements if this variant is an instance of BinaryVariant and an array
+ * @throws IllegalArgumentException If this variant is not an instance of BinaryVariant or not an
+ * array.
+ */
+ public static int arraySize(Variant variant) {
+ Preconditions.checkArgument(
+ variant instanceof BinaryVariant,
+ "Invalid variant implementation instance:%s. Only BinaryVariant is supported.",
+ variant.getClass().getName());
+
+ Preconditions.checkArgument(
+ variant.getType() == Variant.Type.ARRAY,
+ "Invalid variant type:%s. Expecting ARRAY variant type.",
+ variant.getType());
+
+ BinaryVariant binaryArrayVariant = (BinaryVariant) variant;
+ return handleArray(
+ binaryArrayVariant.getValue(), 0, (size, offsetSize, offsetStart, dataStart) -> size);
+ }
+
+ /**
+ * Get the field names of an object variant only at top level. Doesn't include the nested fields.
+ *
+ * @return List of field names if this is an instance of BinaryVariant and is an object
+ * @throws IllegalArgumentException If this variant is not an instance of BinaryVariant or not an
+ * object.
+ */
+ public static List<String> fieldNames(Variant variant) {
+ Preconditions.checkArgument(
+ variant instanceof BinaryVariant,
+ "Invalid variant implementation instance:%s. Only BinaryVariant is supported.",
+ variant.getClass().getName());
+
+ Preconditions.checkArgument(
+ variant.getType() == Variant.Type.OBJECT,
+ "Invalid variant type:%s. Expecting OBJECT variant type.",
+ variant.getType());
+
+ BinaryVariant binaryArrayVariant = (BinaryVariant) variant;
+ return BinaryVariantUtil.handleObject(
+ binaryArrayVariant.getValue(),
+ 0,
+ (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ List<String> fieldNames = Lists.newArrayList();
+ for (int i = 0; i < size; i++) {
+ int id = readUnsigned(binaryArrayVariant.getValue(), idStart + idSize * i, idSize);
+ String fieldName = getMetadataKey(binaryArrayVariant.getMetadata(), id);
+ fieldNames.add(fieldName);
+ }
+ return fieldNames;
+ });
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java
new file mode 100644
index 0000000000..18473bf4f1
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.iceberg.hadoop.SerializableConfiguration;
+import org.apache.iceberg.hive.HiveCatalog;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.rest.RESTCatalog;
+
+/** Serializable loader to load an Iceberg {@link Catalog}. */
+public interface CatalogLoader extends Serializable, Cloneable {
+
+ /**
+ * Create a new catalog with the provided properties. NOTICE: for flink, we may initialize the
+ * {@link CatalogLoader} at flink sql client side or job manager side, and then serialize this
+ * catalog loader to task manager, finally deserialize it and create a new catalog at task manager
+ * side.
+ *
+ * @return a newly created {@link Catalog}
+ */
+ Catalog loadCatalog();
+
+ /** Clone a CatalogLoader. */
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ CatalogLoader clone();
+
+ static CatalogLoader hadoop(
+ String name, Configuration hadoopConf, Map<String, String> properties) {
+ return new HadoopCatalogLoader(name, hadoopConf, properties);
+ }
+
+ static CatalogLoader hive(String name, Configuration hadoopConf, Map<String, String> properties) {
+ return new HiveCatalogLoader(name, hadoopConf, properties);
+ }
+
+ static CatalogLoader rest(String name, Configuration hadoopConf, Map<String, String> properties) {
+ return new RESTCatalogLoader(name, hadoopConf, properties);
+ }
+
+ static CatalogLoader custom(
+ String name, Map<String, String> properties, Configuration hadoopConf, String impl) {
+ return new CustomCatalogLoader(name, properties, hadoopConf, impl);
+ }
+
+ class HadoopCatalogLoader implements CatalogLoader {
+ private final String catalogName;
+ private final SerializableConfiguration hadoopConf;
+ private final String warehouseLocation;
+ private final Map<String, String> properties;
+
+ private HadoopCatalogLoader(
+ String catalogName, Configuration conf, Map<String, String> properties) {
+ this.catalogName = catalogName;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.warehouseLocation = properties.get(CatalogProperties.WAREHOUSE_LOCATION);
+ this.properties = Maps.newHashMap(properties);
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(
+ HadoopCatalog.class.getName(), catalogName, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new HadoopCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("catalogName", catalogName)
+ .add("warehouseLocation", warehouseLocation)
+ .toString();
+ }
+ }
+
+ class HiveCatalogLoader implements CatalogLoader {
+ private final String catalogName;
+ private final SerializableConfiguration hadoopConf;
+ private final String uri;
+ private final String warehouse;
+ private final int clientPoolSize;
+ private final Map<String, String> properties;
+
+ private HiveCatalogLoader(
+ String catalogName, Configuration conf, Map<String, String> properties) {
+ this.catalogName = catalogName;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.uri = properties.get(CatalogProperties.URI);
+ this.warehouse = properties.get(CatalogProperties.WAREHOUSE_LOCATION);
+ this.clientPoolSize =
+ properties.containsKey(CatalogProperties.CLIENT_POOL_SIZE)
+ ? Integer.parseInt(properties.get(CatalogProperties.CLIENT_POOL_SIZE))
+ : CatalogProperties.CLIENT_POOL_SIZE_DEFAULT;
+ this.properties = Maps.newHashMap(properties);
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(
+ HiveCatalog.class.getName(), catalogName, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new HiveCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("catalogName", catalogName)
+ .add("uri", uri)
+ .add("warehouse", warehouse)
+ .add("clientPoolSize", clientPoolSize)
+ .toString();
+ }
+ }
+
+ class RESTCatalogLoader implements CatalogLoader {
+ private final String catalogName;
+ private final SerializableConfiguration hadoopConf;
+ private final Map<String, String> properties;
+
+ private RESTCatalogLoader(
+ String catalogName, Configuration conf, Map<String, String> properties) {
+ this.catalogName = catalogName;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.properties = Maps.newHashMap(properties);
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(
+ RESTCatalog.class.getName(), catalogName, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new RESTCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("catalogName", catalogName)
+ .add("properties", properties)
+ .toString();
+ }
+ }
+
+ class CustomCatalogLoader implements CatalogLoader {
+
+ private final SerializableConfiguration hadoopConf;
+ private final Map<String, String> properties;
+ private final String name;
+ private final String impl;
+
+ private CustomCatalogLoader(
+ String name, Map<String, String> properties, Configuration conf, String impl) {
+ this.hadoopConf = new SerializableConfiguration(conf);
+ this.properties = Maps.newHashMap(properties); // wrap into a hashmap for serialization
+ this.name = name;
+ this.impl =
+ Preconditions.checkNotNull(
+ impl, "Cannot initialize custom Catalog, impl class name is null");
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ return CatalogUtil.loadCatalog(impl, name, properties, hadoopConf.get());
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return new CustomCatalogLoader(name, properties, new Configuration(hadoopConf.get()), impl);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("name", name).add("impl", impl).toString();
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
new file mode 100644
index 0000000000..1d1505a28c
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
@@ -0,0 +1,879 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.catalog.AbstractCatalog;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogDatabase;
+import org.apache.flink.table.catalog.CatalogDatabaseImpl;
+import org.apache.flink.table.catalog.CatalogFunction;
+import org.apache.flink.table.catalog.CatalogPartition;
+import org.apache.flink.table.catalog.CatalogPartitionSpec;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.catalog.TableChange;
+import org.apache.flink.table.catalog.exceptions.CatalogException;
+import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
+import org.apache.flink.table.catalog.exceptions.FunctionNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException;
+import org.apache.flink.table.catalog.stats.CatalogColumnStatistics;
+import org.apache.flink.table.catalog.stats.CatalogTableStatistics;
+import org.apache.flink.table.expressions.Expression;
+import org.apache.flink.table.factories.Factory;
+import org.apache.flink.util.StringUtils;
+import org.apache.iceberg.CachingCatalog;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.MetadataTableType;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.flink.util.FlinkAlterTableUtil;
+import org.apache.iceberg.flink.util.FlinkCompatibilityUtil;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Splitter;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+
+/**
+ * A Flink Catalog implementation that wraps an Iceberg {@link Catalog}.
+ *
+ * <p>The mapping between Flink database and Iceberg namespace: Supplying a base namespace for a
+ * given catalog, so if you have a catalog that supports a 2-level namespace, you would supply the
+ * first level in the catalog configuration and the second level would be exposed as Flink
+ * databases.
+ *
+ * <p>The Iceberg table manages its partitions by itself. The partition of the Iceberg table is
+ * independent of the partition of Flink.
+ */
+@Internal
+public class FlinkCatalog extends AbstractCatalog {
+ private final CatalogLoader catalogLoader;
+ private final Catalog icebergCatalog;
+ private final Namespace baseNamespace;
+ private final SupportsNamespaces asNamespaceCatalog;
+ private final Closeable closeable;
+ private final boolean cacheEnabled;
+
+ public FlinkCatalog(
+ String catalogName,
+ String defaultDatabase,
+ Namespace baseNamespace,
+ CatalogLoader catalogLoader,
+ boolean cacheEnabled,
+ long cacheExpirationIntervalMs) {
+ super(catalogName, defaultDatabase);
+ this.catalogLoader = catalogLoader;
+ this.baseNamespace = baseNamespace;
+ this.cacheEnabled = cacheEnabled;
+
+ Catalog originalCatalog = catalogLoader.loadCatalog();
+ icebergCatalog =
+ cacheEnabled
+ ? CachingCatalog.wrap(originalCatalog, cacheExpirationIntervalMs)
+ : originalCatalog;
+ asNamespaceCatalog =
+ originalCatalog instanceof SupportsNamespaces ? (SupportsNamespaces) originalCatalog : null;
+ closeable = originalCatalog instanceof Closeable ? (Closeable) originalCatalog : null;
+
+ FlinkEnvironmentContext.init();
+ }
+
+ @Override
+ public void open() throws CatalogException {}
+
+ @Override
+ public void close() throws CatalogException {
+ if (closeable != null) {
+ try {
+ closeable.close();
+ } catch (IOException e) {
+ throw new CatalogException(e);
+ }
+ }
+ }
+
+ public Catalog catalog() {
+ return icebergCatalog;
+ }
+
+ /** Append a new level to the base namespace */
+ private static Namespace appendLevel(Namespace baseNamespace, String newLevel) {
+ String[] namespace = new String[baseNamespace.levels().length + 1];
+ System.arraycopy(baseNamespace.levels(), 0, namespace, 0, baseNamespace.levels().length);
+ namespace[baseNamespace.levels().length] = newLevel;
+ return Namespace.of(namespace);
+ }
+
+ TableIdentifier toIdentifier(ObjectPath path) {
+ String objectName = path.getObjectName();
+ List<String> tableName = Splitter.on('$').splitToList(objectName);
+
+ if (tableName.size() == 1) {
+ return TableIdentifier.of(
+ appendLevel(baseNamespace, path.getDatabaseName()), path.getObjectName());
+ } else if (tableName.size() == 2 && MetadataTableType.from(tableName.get(1)) != null) {
+ return TableIdentifier.of(
+ appendLevel(appendLevel(baseNamespace, path.getDatabaseName()), tableName.get(0)),
+ tableName.get(1));
+ } else {
+ throw new IllegalArgumentException("Illegal table name:" + objectName);
+ }
+ }
+
+ @Override
+ public List<String> listDatabases() throws CatalogException {
+ if (asNamespaceCatalog == null) {
+ return Collections.singletonList(getDefaultDatabase());
+ }
+
+ return asNamespaceCatalog.listNamespaces(baseNamespace).stream()
+ .map(n -> n.level(n.levels().length - 1))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public CatalogDatabase getDatabase(String databaseName)
+ throws DatabaseNotExistException, CatalogException {
+ if (asNamespaceCatalog == null) {
+ if (!getDefaultDatabase().equals(databaseName)) {
+ throw new DatabaseNotExistException(getName(), databaseName);
+ } else {
+ return new CatalogDatabaseImpl(Maps.newHashMap(), "");
+ }
+ } else {
+ try {
+ Map<String, String> metadata =
+ Maps.newHashMap(
+ asNamespaceCatalog.loadNamespaceMetadata(appendLevel(baseNamespace, databaseName)));
+ String comment = metadata.remove("comment");
+ return new CatalogDatabaseImpl(metadata, comment);
+ } catch (NoSuchNamespaceException e) {
+ throw new DatabaseNotExistException(getName(), databaseName, e);
+ }
+ }
+ }
+
+ @Override
+ public boolean databaseExists(String databaseName) throws CatalogException {
+ try {
+ getDatabase(databaseName);
+ return true;
+ } catch (DatabaseNotExistException ignore) {
+ return false;
+ }
+ }
+
+ @Override
+ public void createDatabase(String name, CatalogDatabase database, boolean ignoreIfExists)
+ throws DatabaseAlreadyExistException, CatalogException {
+ createDatabase(
+ name, mergeComment(database.getProperties(), database.getComment()), ignoreIfExists);
+ }
+
+ private void createDatabase(
+ String databaseName, Map<String, String> metadata, boolean ignoreIfExists)
+ throws DatabaseAlreadyExistException, CatalogException {
+ if (asNamespaceCatalog != null) {
+ try {
+ asNamespaceCatalog.createNamespace(appendLevel(baseNamespace, databaseName), metadata);
+ } catch (AlreadyExistsException e) {
+ if (!ignoreIfExists) {
+ throw new DatabaseAlreadyExistException(getName(), databaseName, e);
+ }
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ "Namespaces are not supported by catalog: " + getName());
+ }
+ }
+
+ private Map<String, String> mergeComment(Map<String, String> metadata, String comment) {
+ Map<String, String> ret = Maps.newHashMap(metadata);
+ if (metadata.containsKey("comment")) {
+ throw new CatalogException("Database properties should not contain key: 'comment'.");
+ }
+
+ if (!StringUtils.isNullOrWhitespaceOnly(comment)) {
+ ret.put("comment", comment);
+ }
+ return ret;
+ }
+
+ @Override
+ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade)
+ throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException {
+ if (asNamespaceCatalog != null) {
+ try {
+ boolean success = asNamespaceCatalog.dropNamespace(appendLevel(baseNamespace, name));
+ if (!success && !ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name);
+ }
+ } catch (NoSuchNamespaceException e) {
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name, e);
+ }
+ } catch (NamespaceNotEmptyException e) {
+ throw new DatabaseNotEmptyException(getName(), name, e);
+ }
+ } else {
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name);
+ }
+ }
+ }
+
+ @Override
+ public void alterDatabase(String name, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
+ throws DatabaseNotExistException, CatalogException {
+ if (asNamespaceCatalog != null) {
+ Namespace namespace = appendLevel(baseNamespace, name);
+ Map<String, String> updates = Maps.newHashMap();
+ Set<String> removals = Sets.newHashSet();
+
+ try {
+ Map<String, String> oldProperties = asNamespaceCatalog.loadNamespaceMetadata(namespace);
+ Map<String, String> newProperties =
+ mergeComment(newDatabase.getProperties(), newDatabase.getComment());
+
+ for (String key : oldProperties.keySet()) {
+ if (!newProperties.containsKey(key)) {
+ removals.add(key);
+ }
+ }
+
+ for (Map.Entry<String, String> entry : newProperties.entrySet()) {
+ if (!entry.getValue().equals(oldProperties.get(entry.getKey()))) {
+ updates.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ if (!updates.isEmpty()) {
+ asNamespaceCatalog.setProperties(namespace, updates);
+ }
+
+ if (!removals.isEmpty()) {
+ asNamespaceCatalog.removeProperties(namespace, removals);
+ }
+
+ } catch (NoSuchNamespaceException e) {
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name, e);
+ }
+ }
+ } else {
+ if (getDefaultDatabase().equals(name)) {
+ throw new CatalogException(
+ "Can not alter the default database when the iceberg catalog doesn't support namespaces.");
+ }
+ if (!ignoreIfNotExists) {
+ throw new DatabaseNotExistException(getName(), name);
+ }
+ }
+ }
+
+ @Override
+ public List<String> listTables(String databaseName)
+ throws DatabaseNotExistException, CatalogException {
+ try {
+ return icebergCatalog.listTables(appendLevel(baseNamespace, databaseName)).stream()
+ .map(TableIdentifier::name)
+ .collect(Collectors.toList());
+ } catch (NoSuchNamespaceException e) {
+ throw new DatabaseNotExistException(getName(), databaseName, e);
+ }
+ }
+
+ @Override
+ public CatalogTable getTable(ObjectPath tablePath)
+ throws TableNotExistException, CatalogException {
+ Table table = loadIcebergTable(tablePath);
+
+ // Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table.
+ // As Flink API accepts only Map<String, String> for props, here we are serializing catalog
+ // name, database, table as json string to distinguish between catalog info
+ // and table properties in createTable.
+ String srcCatalogProps =
+ FlinkCreateTableOptions.toJson(
+ getName(), tablePath.getDatabaseName(), tablePath.getObjectName());
+
+ Map<String, String> tableProps = table.properties();
+ if (tableProps.containsKey(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY)
+ || tableProps.containsKey(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Source table %s contains one/all of the reserved property keys: %s, %s.",
+ tablePath,
+ FlinkCreateTableOptions.CONNECTOR_PROPS_KEY,
+ FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY));
+ }
+
+ ImmutableMap.Builder<String, String> mergedProps = ImmutableMap.builder();
+ mergedProps.put(
+ FlinkCreateTableOptions.CONNECTOR_PROPS_KEY, FlinkDynamicTableFactory.FACTORY_IDENTIFIER);
+ mergedProps.put(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY, srcCatalogProps);
+ mergedProps.putAll(tableProps);
+
+ return toCatalogTableWithProps(table, mergedProps.build());
+ }
+
+ private Table loadIcebergTable(ObjectPath tablePath) throws TableNotExistException {
+ try {
+ Table table = icebergCatalog.loadTable(toIdentifier(tablePath));
+ if (cacheEnabled) {
+ table.refresh();
+ }
+
+ return table;
+ } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
+ throw new TableNotExistException(getName(), tablePath, e);
+ }
+ }
+
+ @Override
+ public boolean tableExists(ObjectPath tablePath) throws CatalogException {
+ return icebergCatalog.tableExists(toIdentifier(tablePath));
+ }
+
+ @Override
+ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ try {
+ icebergCatalog.dropTable(toIdentifier(tablePath));
+ } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
+ if (!ignoreIfNotExists) {
+ throw new TableNotExistException(getName(), tablePath, e);
+ }
+ }
+ }
+
+ @Override
+ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
+ throws TableNotExistException, TableAlreadyExistException, CatalogException {
+ try {
+ icebergCatalog.renameTable(
+ toIdentifier(tablePath),
+ toIdentifier(new ObjectPath(tablePath.getDatabaseName(), newTableName)));
+ } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
+ if (!ignoreIfNotExists) {
+ throw new TableNotExistException(getName(), tablePath, e);
+ }
+ } catch (AlreadyExistsException e) {
+ throw new TableAlreadyExistException(getName(), tablePath, e);
+ }
+ }
+
+ @Override
+ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
+ throws CatalogException, TableAlreadyExistException {
+ // Creating Iceberg table using connector is allowed only when table is created using LIKE
+ if (Objects.equals(
+ table.getOptions().get(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY),
+ FlinkDynamicTableFactory.FACTORY_IDENTIFIER)
+ && table.getOptions().get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY) == null) {
+ throw new IllegalArgumentException(
+ "Cannot create the table with 'connector'='iceberg' table property in "
+ + "an iceberg catalog, Please create table with 'connector'='iceberg' property in a non-iceberg catalog or "
+ + "create table without 'connector'='iceberg' related properties in an iceberg table.");
+ }
+
+ Preconditions.checkArgument(
+ table instanceof ResolvedCatalogTable,
+ "Expected a ResolvedCatalogTable but got: %s. "
+ + "Iceberg Flink catalog only supports resolved catalog tables "
+ + "(Materialized tables and other table kinds are not supported).",
+ table == null ? "null" : table.getClass().getName());
+ createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists);
+ }
+
+ void createIcebergTable(ObjectPath tablePath, ResolvedCatalogTable table, boolean ignoreIfExists)
+ throws CatalogException, TableAlreadyExistException {
+ validateFlinkTable(table);
+
+ Schema icebergSchema = FlinkSchemaUtil.convert(table.getResolvedSchema());
+ PartitionSpec spec = toPartitionSpec(table.getPartitionKeys(), icebergSchema);
+ ImmutableMap.Builder<String, String> properties = ImmutableMap.builder();
+ String location = null;
+ for (Map.Entry<String, String> entry : table.getOptions().entrySet()) {
+ if (!isReservedProperty(entry.getKey())) {
+ properties.put(entry.getKey(), entry.getValue());
+ } else {
+ // Filtering reserved properties like catalog properties(added to support CREATE TABLE LIKE
+ // in getTable()), location and not persisting on table properties.
+ if (FlinkCreateTableOptions.LOCATION_KEY.equalsIgnoreCase(entry.getKey())) {
+ location = entry.getValue();
+ }
+ }
+ }
+
+ String comment = table.getComment();
+ if (comment != null && !comment.isEmpty()) {
+ properties.put(TableProperties.COMMENT, comment);
+ }
+
+ try {
+ icebergCatalog.createTable(
+ toIdentifier(tablePath), icebergSchema, spec, location, properties.build());
+ } catch (AlreadyExistsException e) {
+ if (!ignoreIfExists) {
+ throw new TableAlreadyExistException(getName(), tablePath, e);
+ }
+ }
+ }
+
+ private boolean isReservedProperty(String prop) {
+ return FlinkCreateTableOptions.LOCATION_KEY.equalsIgnoreCase(prop)
+ || FlinkCreateTableOptions.CONNECTOR_PROPS_KEY.equalsIgnoreCase(prop)
+ || FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY.equalsIgnoreCase(prop);
+ }
+
+ private static void validateTableSchemaAndPartition(CatalogTable ct1, CatalogTable ct2) {
+ if (!Objects.equals(ct1.getUnresolvedSchema(), ct2.getUnresolvedSchema())) {
+ throw new UnsupportedOperationException(
+ "Altering schema is not supported in the old alterTable API. "
+ + "To alter schema, use the other alterTable API and provide a list of TableChange's.");
+ }
+
+ validateTablePartition(ct1, ct2);
+ }
+
+ private static void validateTablePartition(CatalogTable ct1, CatalogTable ct2) {
+ if (!ct1.getPartitionKeys().equals(ct2.getPartitionKeys())) {
+ throw new UnsupportedOperationException("Altering partition keys is not supported yet.");
+ }
+ }
+
+ /**
+ * This alterTable API only supports altering table properties.
+ *
+ * <p>Support for adding/removing/renaming columns cannot be done by comparing CatalogTable
+ * instances, unless the Flink schema contains Iceberg column IDs.
+ *
+ * <p>To alter columns, use the other alterTable API and provide a list of TableChange's.
+ *
+ * @param tablePath path of the table or view to be modified
+ * @param newTable the new table definition
+ * @param ignoreIfNotExists flag to specify behavior when the table or view does not exist: if set
+ * to false, throw an exception, if set to true, do nothing.
+ * @throws CatalogException in case of any runtime exception
+ * @throws TableNotExistException if the table does not exist
+ */
+ @Override
+ public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
+ throws CatalogException, TableNotExistException {
+ validateFlinkTable(newTable);
+
+ Table icebergTable;
+ try {
+ icebergTable = loadIcebergTable(tablePath);
+ } catch (TableNotExistException e) {
+ if (!ignoreIfNotExists) {
+ throw e;
+ } else {
+ return;
+ }
+ }
+
+ CatalogTable table = toCatalogTable(icebergTable);
+ validateTableSchemaAndPartition(table, (CatalogTable) newTable);
+
+ Map<String, String> oldProperties = table.getOptions();
+ Map<String, String> setProperties = Maps.newHashMap();
+
+ String setLocation = null;
+ String setSnapshotId = null;
+ String pickSnapshotId = null;
+
+ for (Map.Entry<String, String> entry : newTable.getOptions().entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (Objects.equals(value, oldProperties.get(key))) {
+ continue;
+ }
+
+ if (FlinkCreateTableOptions.LOCATION_KEY.equalsIgnoreCase(key)) {
+ setLocation = value;
+ } else if ("current-snapshot-id".equalsIgnoreCase(key)) {
+ setSnapshotId = value;
+ } else if ("cherry-pick-snapshot-id".equalsIgnoreCase(key)) {
+ pickSnapshotId = value;
+ } else {
+ setProperties.put(key, value);
+ }
+ }
+
+ oldProperties
+ .keySet()
+ .forEach(
+ k -> {
+ if (!newTable.getOptions().containsKey(k)) {
+ setProperties.put(k, null);
+ }
+ });
+
+ FlinkAlterTableUtil.commitChanges(
+ icebergTable, setLocation, setSnapshotId, pickSnapshotId, setProperties);
+ }
+
+ @Override
+ public void alterTable(
+ ObjectPath tablePath,
+ CatalogBaseTable newTable,
+ List<TableChange> tableChanges,
+ boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ validateFlinkTable(newTable);
+
+ Table icebergTable;
+ try {
+ icebergTable = loadIcebergTable(tablePath);
+ } catch (TableNotExistException e) {
+ if (!ignoreIfNotExists) {
+ throw e;
+ } else {
+ return;
+ }
+ }
+
+ // Does not support altering partition yet.
+ validateTablePartition(toCatalogTable(icebergTable), (CatalogTable) newTable);
+
+ String setLocation = null;
+ String setSnapshotId = null;
+ String cherrypickSnapshotId = null;
+
+ List<TableChange> propertyChanges = Lists.newArrayList();
+ List<TableChange> schemaChanges = Lists.newArrayList();
+ for (TableChange change : tableChanges) {
+ if (change instanceof TableChange.SetOption) {
+ TableChange.SetOption set = (TableChange.SetOption) change;
+
+ if (FlinkCreateTableOptions.LOCATION_KEY.equalsIgnoreCase(set.getKey())) {
+ setLocation = set.getValue();
+ } else if ("current-snapshot-id".equalsIgnoreCase(set.getKey())) {
+ setSnapshotId = set.getValue();
+ } else if ("cherry-pick-snapshot-id".equalsIgnoreCase(set.getKey())) {
+ cherrypickSnapshotId = set.getValue();
+ } else {
+ propertyChanges.add(change);
+ }
+ } else if (change instanceof TableChange.ResetOption) {
+ propertyChanges.add(change);
+ } else {
+ schemaChanges.add(change);
+ }
+ }
+
+ FlinkAlterTableUtil.commitChanges(
+ icebergTable,
+ setLocation,
+ setSnapshotId,
+ cherrypickSnapshotId,
+ schemaChanges,
+ propertyChanges);
+ }
+
+ private static void validateFlinkTable(CatalogBaseTable table) {
+ Preconditions.checkArgument(
+ table instanceof CatalogTable, "The Table should be a CatalogTable.");
+
+ org.apache.flink.table.api.Schema schema = table.getUnresolvedSchema();
+ schema
+ .getColumns()
+ .forEach(
+ column -> {
+ if (!FlinkCompatibilityUtil.isPhysicalColumn(column)) {
+ throw new UnsupportedOperationException(
+ "Creating table with computed columns is not supported yet.");
+ }
+ });
+
+ if (!schema.getWatermarkSpecs().isEmpty()) {
+ throw new UnsupportedOperationException(
+ "Creating table with watermark specs is not supported yet.");
+ }
+ }
+
+ private static PartitionSpec toPartitionSpec(List<String> partitionKeys, Schema icebergSchema) {
+ PartitionSpec.Builder builder = PartitionSpec.builderFor(icebergSchema);
+ partitionKeys.forEach(builder::identity);
+ return builder.build();
+ }
+
+ private static List<String> toPartitionKeys(PartitionSpec spec, Schema icebergSchema) {
+ ImmutableList.Builder<String> partitionKeysBuilder = ImmutableList.builder();
+ for (PartitionField field : spec.fields()) {
+ if (field.transform().isIdentity()) {
+ partitionKeysBuilder.add(icebergSchema.findColumnName(field.sourceId()));
+ } else {
+ // Not created by Flink SQL.
+ // For compatibility with iceberg tables, return empty.
+ // TODO modify this after Flink support partition transform.
+ return Collections.emptyList();
+ }
+ }
+ return partitionKeysBuilder.build();
+ }
+
+ static CatalogTable toCatalogTableWithProps(Table table, Map<String, String> props) {
+ ResolvedSchema resolvedSchema = FlinkSchemaUtil.toResolvedSchema(table.schema());
+ List<String> partitionKeys = toPartitionKeys(table.spec(), table.schema());
+
+ // NOTE: We can not create a IcebergCatalogTable extends CatalogTable, because Flink optimizer
+ // may use DefaultCatalogTable to copy a new catalog table.
+ // Let's re-loading table from Iceberg catalog when creating source/sink operators.
+ return CatalogTable.newBuilder()
+ .schema(
+ org.apache.flink.table.api.Schema.newBuilder()
+ .fromResolvedSchema(resolvedSchema)
+ .build())
+ .partitionKeys(partitionKeys)
+ .options(props)
+ .build();
+ }
+
+ static CatalogTable toCatalogTable(Table table) {
+ return toCatalogTableWithProps(table, table.properties());
+ }
+
+ @Override
+ public Optional<Factory> getFactory() {
+ return Optional.of(new FlinkDynamicTableFactory(this));
+ }
+
+ CatalogLoader getCatalogLoader() {
+ return catalogLoader;
+ }
+
+ // ------------------------------ Unsupported methods
+ // ---------------------------------------------
+
+ @Override
+ public List<String> listViews(String databaseName) throws CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void createPartition(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogPartition partition,
+ boolean ignoreIfExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropPartition(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartition(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogPartition newPartition,
+ boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<String> listFunctions(String dbName) throws CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CatalogFunction getFunction(ObjectPath functionPath)
+ throws FunctionNotExistException, CatalogException {
+ throw new FunctionNotExistException(getName(), functionPath);
+ }
+
+ @Override
+ public boolean functionExists(ObjectPath functionPath) throws CatalogException {
+ return false;
+ }
+
+ @Override
+ public void createFunction(
+ ObjectPath functionPath, CatalogFunction function, boolean ignoreIfExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterFunction(
+ ObjectPath functionPath, CatalogFunction newFunction, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterTableStatistics(
+ ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterTableColumnStatistics(
+ ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartitionStatistics(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogTableStatistics partitionStatistics,
+ boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartitionColumnStatistics(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogColumnStatistics columnStatistics,
+ boolean ignoreIfNotExists)
+ throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
+ throws TableNotExistException, TableNotPartitionedException, CatalogException {
+ Table table = loadIcebergTable(tablePath);
+
+ if (table.spec().isUnpartitioned()) {
+ throw new TableNotPartitionedException(icebergCatalog.name(), tablePath);
+ }
+
+ Set<CatalogPartitionSpec> set = Sets.newHashSet();
+ try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
+ for (DataFile dataFile : CloseableIterable.transform(tasks, FileScanTask::file)) {
+ Map<String, String> map = Maps.newHashMap();
+ StructLike structLike = dataFile.partition();
+ PartitionSpec spec = table.specs().get(dataFile.specId());
+ for (int i = 0; i < structLike.size(); i++) {
+ map.put(spec.fields().get(i).name(), String.valueOf(structLike.get(i, Object.class)));
+ }
+ set.add(new CatalogPartitionSpec(map));
+ }
+ } catch (IOException e) {
+ throw new CatalogException(
+ String.format("Failed to list partitions of table %s", tablePath), e);
+ }
+
+ return Lists.newArrayList(set);
+ }
+
+ @Override
+ public List<CatalogPartitionSpec> listPartitions(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<CatalogPartitionSpec> listPartitionsByFilter(
+ ObjectPath tablePath, List<Expression> filters) throws CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ // After partition pruning and filter push down, the statistics have become very inaccurate, so
+ // the statistics from
+ // here are of little significance.
+ // Flink will support something like SupportsReportStatistics in future.
+
+ @Override
+ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws CatalogException {
+ return CatalogTableStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath)
+ throws CatalogException {
+ return CatalogColumnStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogTableStatistics getPartitionStatistics(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
+ return CatalogTableStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogColumnStatistics getPartitionColumnStatistics(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
+ return CatalogColumnStatistics.UNKNOWN;
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java
new file mode 100644
index 0000000000..24e2bdbba3
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.runtime.util.HadoopUtils;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.factories.CatalogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.util.PropertyUtil;
+
+/**
+ * A Flink Catalog factory implementation that creates {@link FlinkCatalog}.
+ *
+ * <p>This supports the following catalog configuration options:
+ *
+ * <ul>
+ * <li><code>type</code> - Flink catalog factory key, should be "iceberg"
+ * <li><code>catalog-type</code> - iceberg catalog type, "hive", "hadoop" or "rest"
+ * <li><code>uri</code> - the Hive Metastore URI (Hive catalog only)
+ * <li><code>clients</code> - the Hive Client Pool Size (Hive catalog only)
+ * <li><code>warehouse</code> - the warehouse path (Hadoop catalog only)
+ * <li><code>default-database</code> - a database name to use as the default
+ * <li><code>base-namespace</code> - a base namespace as the prefix for all databases (Hadoop
+ * catalog only)
+ * <li><code>cache-enabled</code> - whether to enable catalog cache
+ * </ul>
+ *
+ * <p>To use a custom catalog that is not a Hive or Hadoop catalog, extend this class and override
+ * {@link #createCatalogLoader(String, Map, Configuration)}.
+ */
+public class FlinkCatalogFactory implements CatalogFactory {
+
+ public static final String FACTORY_IDENTIFIER = "iceberg";
+
+ // Can not just use "type", it conflicts with CATALOG_TYPE.
+ public static final String ICEBERG_CATALOG_TYPE = "catalog-type";
+ public static final String ICEBERG_CATALOG_TYPE_HADOOP = "hadoop";
+ public static final String ICEBERG_CATALOG_TYPE_HIVE = "hive";
+ public static final String ICEBERG_CATALOG_TYPE_REST = "rest";
+
+ public static final String HIVE_CONF_DIR = "hive-conf-dir";
+ public static final String HADOOP_CONF_DIR = "hadoop-conf-dir";
+ public static final String DEFAULT_DATABASE = "default-database";
+ public static final String DEFAULT_DATABASE_NAME = "default";
+ public static final String DEFAULT_CATALOG_NAME = "default_catalog";
+ public static final String BASE_NAMESPACE = "base-namespace";
+
+ /**
+ * Create an Iceberg {@link org.apache.iceberg.catalog.Catalog} loader to be used by this Flink
+ * catalog adapter.
+ *
+ * @param name Flink's catalog name
+ * @param properties Flink's catalog properties
+ * @param hadoopConf Hadoop configuration for catalog
+ * @return an Iceberg catalog loader
+ */
+ static CatalogLoader createCatalogLoader(
+ String name, Map<String, String> properties, Configuration hadoopConf) {
+ String catalogImpl = properties.get(CatalogProperties.CATALOG_IMPL);
+ if (catalogImpl != null) {
+ String catalogType = properties.get(ICEBERG_CATALOG_TYPE);
+ Preconditions.checkArgument(
+ catalogType == null,
+ "Cannot create catalog %s, both catalog-type and catalog-impl are set: catalog-type=%s, catalog-impl=%s",
+ name,
+ catalogType,
+ catalogImpl);
+ return CatalogLoader.custom(name, properties, hadoopConf, catalogImpl);
+ }
+
+ String catalogType = properties.getOrDefault(ICEBERG_CATALOG_TYPE, ICEBERG_CATALOG_TYPE_HIVE);
+ switch (catalogType.toLowerCase(Locale.ROOT)) {
+ case ICEBERG_CATALOG_TYPE_HIVE:
+ // The values of properties 'uri', 'warehouse', 'hive-conf-dir' are allowed to be null, in
+ // that case it will
+ // fallback to parse those values from hadoop configuration which is loaded from classpath.
+ String hiveConfDir = properties.get(HIVE_CONF_DIR);
+ String hadoopConfDir = properties.get(HADOOP_CONF_DIR);
+ Configuration newHadoopConf = mergeHiveConf(hadoopConf, hiveConfDir, hadoopConfDir);
+ return CatalogLoader.hive(name, newHadoopConf, properties);
+
+ case ICEBERG_CATALOG_TYPE_HADOOP:
+ return CatalogLoader.hadoop(name, hadoopConf, properties);
+
+ case ICEBERG_CATALOG_TYPE_REST:
+ return CatalogLoader.rest(name, hadoopConf, properties);
+
+ default:
+ throw new UnsupportedOperationException(
+ "Unknown catalog-type: " + catalogType + " (Must be 'hive', 'hadoop' or 'rest')");
+ }
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return FACTORY_IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return ImmutableSet.<ConfigOption<?>>builder().build();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return ImmutableSet.<ConfigOption<?>>builder().build();
+ }
+
+ @Override
+ public Catalog createCatalog(Context context) {
+ return createCatalog(context.getName(), context.getOptions(), clusterHadoopConf());
+ }
+
+ protected Catalog createCatalog(
+ String name, Map<String, String> properties, Configuration hadoopConf) {
+ CatalogLoader catalogLoader = createCatalogLoader(name, properties, hadoopConf);
+ String defaultDatabase = properties.getOrDefault(DEFAULT_DATABASE, DEFAULT_DATABASE_NAME);
+
+ Namespace baseNamespace = Namespace.empty();
+ if (properties.containsKey(BASE_NAMESPACE)) {
+ baseNamespace = Namespace.of(properties.get(BASE_NAMESPACE).split("\\."));
+ }
+
+ boolean cacheEnabled =
+ PropertyUtil.propertyAsBoolean(
+ properties, CatalogProperties.CACHE_ENABLED, CatalogProperties.CACHE_ENABLED_DEFAULT);
+
+ long cacheExpirationIntervalMs =
+ PropertyUtil.propertyAsLong(
+ properties,
+ CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS,
+ CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS_OFF);
+ Preconditions.checkArgument(
+ cacheExpirationIntervalMs != 0,
+ "%s is not allowed to be 0.",
+ CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS);
+
+ return new FlinkCatalog(
+ name,
+ defaultDatabase,
+ baseNamespace,
+ catalogLoader,
+ cacheEnabled,
+ cacheExpirationIntervalMs);
+ }
+
+ private static Configuration mergeHiveConf(
+ Configuration hadoopConf, String hiveConfDir, String hadoopConfDir) {
+ Configuration newConf = new Configuration(hadoopConf);
+ if (!Strings.isNullOrEmpty(hiveConfDir)) {
+ Preconditions.checkState(
+ Files.exists(Paths.get(hiveConfDir, "hive-site.xml")),
+ "There should be a hive-site.xml file under the directory %s",
+ hiveConfDir);
+ newConf.addResource(new Path(hiveConfDir, "hive-site.xml"));
+ } else {
+ // If don't provide the hive-site.xml path explicitly, it will try to load resource from
+ // classpath. If still
+ // couldn't load the configuration file, then it will throw exception in HiveCatalog.
+ URL configFile = CatalogLoader.class.getClassLoader().getResource("hive-site.xml");
+ if (configFile != null) {
+ newConf.addResource(configFile);
+ }
+ }
+
+ if (!Strings.isNullOrEmpty(hadoopConfDir)) {
+ Preconditions.checkState(
+ Files.exists(Paths.get(hadoopConfDir, "hdfs-site.xml")),
+ "Failed to load Hadoop configuration: missing %s",
+ Paths.get(hadoopConfDir, "hdfs-site.xml"));
+ newConf.addResource(new Path(hadoopConfDir, "hdfs-site.xml"));
+ Preconditions.checkState(
+ Files.exists(Paths.get(hadoopConfDir, "core-site.xml")),
+ "Failed to load Hadoop configuration: missing %s",
+ Paths.get(hadoopConfDir, "core-site.xml"));
+ newConf.addResource(new Path(hadoopConfDir, "core-site.xml"));
+ }
+
+ return newConf;
+ }
+
+ public static Configuration clusterHadoopConf() {
+ return HadoopUtils.getHadoopConfiguration(GlobalConfiguration.loadConfiguration());
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java
new file mode 100644
index 0000000000..7661372c88
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.util.TimeUtils;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+
+@Internal
+public class FlinkConfParser {
+
+ private final Map<String, String> tableProperties;
+ private final Map<String, String> options;
+ private final ReadableConfig readableConfig;
+
+ public FlinkConfParser(Table table, Map<String, String> options, ReadableConfig readableConfig) {
+ this.tableProperties = table.properties();
+ this.options = options;
+ this.readableConfig = readableConfig;
+ }
+
+ public FlinkConfParser(Map<String, String> options, ReadableConfig readableConfig) {
+ this.tableProperties = ImmutableMap.of();
+ this.options = options;
+ this.readableConfig = readableConfig;
+ }
+
+ public BooleanConfParser booleanConf() {
+ return new BooleanConfParser();
+ }
+
+ public IntConfParser intConf() {
+ return new IntConfParser();
+ }
+
+ public LongConfParser longConf() {
+ return new LongConfParser();
+ }
+
+ public DoubleConfParser doubleConf() {
+ return new DoubleConfParser();
+ }
+
+ public <E extends Enum<E>> EnumConfParser<E> enumConfParser(Class<E> enumClass) {
+ return new EnumConfParser<>(enumClass);
+ }
+
+ public StringConfParser stringConf() {
+ return new StringConfParser();
+ }
+
+ public DurationConfParser durationConf() {
+ return new DurationConfParser();
+ }
+
+ public class BooleanConfParser extends ConfParser<BooleanConfParser, Boolean> {
+ private Boolean defaultValue;
+
+ @Override
+ protected BooleanConfParser self() {
+ return this;
+ }
+
+ public BooleanConfParser defaultValue(boolean value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public BooleanConfParser defaultValue(String value) {
+ this.defaultValue = Boolean.parseBoolean(value);
+ return self();
+ }
+
+ public boolean parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Boolean::parseBoolean, defaultValue);
+ }
+ }
+
+ public class IntConfParser extends ConfParser<IntConfParser, Integer> {
+ private Integer defaultValue;
+
+ @Override
+ protected IntConfParser self() {
+ return this;
+ }
+
+ public IntConfParser defaultValue(int value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public int parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Integer::parseInt, defaultValue);
+ }
+
+ public Integer parseOptional() {
+ return parse(Integer::parseInt, null);
+ }
+ }
+
+ public class LongConfParser extends ConfParser<LongConfParser, Long> {
+ private Long defaultValue;
+
+ @Override
+ protected LongConfParser self() {
+ return this;
+ }
+
+ public LongConfParser defaultValue(long value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public long parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Long::parseLong, defaultValue);
+ }
+
+ public Long parseOptional() {
+ return parse(Long::parseLong, null);
+ }
+ }
+
+ public class DoubleConfParser extends ConfParser<DoubleConfParser, Double> {
+ private Double defaultValue;
+
+ @Override
+ protected DoubleConfParser self() {
+ return this;
+ }
+
+ public DoubleConfParser defaultValue(double value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public double parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Double::parseDouble, defaultValue);
+ }
+
+ public Double parseOptional() {
+ return parse(Double::parseDouble, null);
+ }
+ }
+
+ public class StringConfParser extends ConfParser<StringConfParser, String> {
+ private String defaultValue;
+
+ @Override
+ protected StringConfParser self() {
+ return this;
+ }
+
+ public StringConfParser defaultValue(String value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public String parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(Function.identity(), defaultValue);
+ }
+
+ public String parseOptional() {
+ return parse(Function.identity(), null);
+ }
+ }
+
+ public class EnumConfParser<E extends Enum<E>> extends ConfParser<EnumConfParser<E>, E> {
+ private E defaultValue;
+ private final Class<E> enumClass;
+
+ EnumConfParser(Class<E> enumClass) {
+ this.enumClass = enumClass;
+ }
+
+ @Override
+ protected EnumConfParser<E> self() {
+ return this;
+ }
+
+ public EnumConfParser<E> defaultValue(E value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public E parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(s -> Enum.valueOf(enumClass, s), defaultValue);
+ }
+
+ public E parseOptional() {
+ return parse(s -> Enum.valueOf(enumClass, s), null);
+ }
+ }
+
+ public class DurationConfParser extends ConfParser<DurationConfParser, Duration> {
+ private Duration defaultValue;
+
+ @Override
+ protected DurationConfParser self() {
+ return this;
+ }
+
+ public DurationConfParser defaultValue(Duration value) {
+ this.defaultValue = value;
+ return self();
+ }
+
+ public Duration parse() {
+ Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+ return parse(TimeUtils::parseDuration, defaultValue);
+ }
+
+ public Duration parseOptional() {
+ return parse(TimeUtils::parseDuration, null);
+ }
+ }
+
+ public abstract class ConfParser<ThisT, T> {
+ private final List<String> optionNames = Lists.newArrayList();
+ private String tablePropertyName;
+ private ConfigOption<T> configOption;
+
+ protected abstract ThisT self();
+
+ public ThisT option(String name) {
+ this.optionNames.add(name);
+ return self();
+ }
+
+ public ThisT flinkConfig(ConfigOption<T> newConfigOption) {
+ this.configOption = newConfigOption;
+ return self();
+ }
+
+ public ThisT tableProperty(String name) {
+ this.tablePropertyName = name;
+ return self();
+ }
+
+ protected T parse(Function<String, T> conversion, T defaultValue) {
+ if (!optionNames.isEmpty()) {
+ for (String optionName : optionNames) {
+ String optionValue = options.get(optionName);
+ if (optionValue != null) {
+ return conversion.apply(optionValue);
+ }
+ }
+ }
+
+ if (configOption != null) {
+ T propertyValue = readableConfig.get(configOption);
+ if (propertyValue != null) {
+ return propertyValue;
+ }
+ }
+
+ if (tablePropertyName != null) {
+ String propertyValue = tableProperties.get(tablePropertyName);
+ if (propertyValue != null) {
+ return conversion.apply(propertyValue);
+ }
+ }
+
+ return defaultValue;
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java
new file mode 100644
index 0000000000..97e2c70d34
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.description.Description;
+import org.apache.flink.configuration.description.TextElement;
+import org.apache.flink.table.api.TableEnvironment;
+import org.apache.iceberg.flink.source.assigner.SplitAssignerType;
+import org.apache.iceberg.util.ThreadPools;
+
+/**
+ * When constructing Flink Iceberg source via Java API, configs can be set in {@link Configuration}
+ * passed to source builder. E.g.
+ *
+ * <pre>
+ * configuration.setBoolean(FlinkConfigOptions.TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM, true);
+ * FlinkSource.forRowData()
+ * .flinkConf(configuration)
+ * ...
+ * </pre>
+ *
+ * <p>When using Flink SQL/table API, connector options can be set in Flink's {@link
+ * TableEnvironment}.
+ *
+ * <pre>
+ * TableEnvironment tEnv = createTableEnv();
+ * tEnv.getConfig()
+ * .getConfiguration()
+ * .setBoolean(FlinkConfigOptions.TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM, true);
+ * </pre>
+ */
+public class FlinkConfigOptions {
+
+ private FlinkConfigOptions() {}
+
+ public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM =
+ ConfigOptions.key("table.exec.iceberg.infer-source-parallelism")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "If is false, parallelism of source are set by config.\n"
+ + "If is true, source parallelism is inferred according to splits number.\n");
+
+ public static final ConfigOption<Integer> TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM_MAX =
+ ConfigOptions.key("table.exec.iceberg.infer-source-parallelism.max")
+ .intType()
+ .defaultValue(100)
+ .withDescription("Sets max infer parallelism for source operator.");
+
+ public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_EXPOSE_SPLIT_LOCALITY_INFO =
+ ConfigOptions.key("table.exec.iceberg.expose-split-locality-info")
+ .booleanType()
+ .noDefaultValue()
+ .withDescription(
+ "Expose split host information to use Flink's locality aware split assigner.");
+
+ public static final ConfigOption<Integer> SOURCE_READER_FETCH_BATCH_RECORD_COUNT =
+ ConfigOptions.key("table.exec.iceberg.fetch-batch-record-count")
+ .intType()
+ .defaultValue(2048)
+ .withDescription("The target number of records for Iceberg reader fetch batch.");
+
+ public static final ConfigOption<Integer> TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE =
+ ConfigOptions.key("table.exec.iceberg.worker-pool-size")
+ .intType()
+ .defaultValue(ThreadPools.WORKER_THREAD_POOL_SIZE)
+ .withDescription("The size of workers pool used to plan or scan manifests.");
+
+ public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_USE_FLIP27_SOURCE =
+ ConfigOptions.key("table.exec.iceberg.use-flip27-source")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription("Use the FLIP-27 based Iceberg source implementation.");
+
+ public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_USE_V2_SINK =
+ ConfigOptions.key("table.exec.iceberg.use-v2-sink")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Use the SinkV2 API based Iceberg sink implementation.");
+
+ public static final ConfigOption<SplitAssignerType> TABLE_EXEC_SPLIT_ASSIGNER_TYPE =
+ ConfigOptions.key("table.exec.iceberg.split-assigner-type")
+ .enumType(SplitAssignerType.class)
+ .defaultValue(SplitAssignerType.SIMPLE)
+ .withDescription(
+ Description.builder()
+ .text("Split assigner type that determine how splits are assigned to readers.")
+ .linebreak()
+ .list(
+ TextElement.text(
+ SplitAssignerType.SIMPLE
+ + ": simple assigner that doesn't provide any guarantee on order or locality."))
+ .build());
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java
new file mode 100644
index 0000000000..a08cb19c8c
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.iceberg.util.JsonUtil;
+
+@Internal
+public class FlinkCreateTableOptions {
+ private final String catalogName;
+ private final String catalogDb;
+ private final String catalogTable;
+
+ private FlinkCreateTableOptions(String catalogName, String catalogDb, String catalogTable) {
+ this.catalogName = catalogName;
+ this.catalogDb = catalogDb;
+ this.catalogTable = catalogTable;
+ }
+
+ public static final ConfigOption<String> CATALOG_NAME =
+ ConfigOptions.key("catalog-name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Catalog name");
+
+ public static final ConfigOption<String> CATALOG_TYPE =
+ ConfigOptions.key(FlinkCatalogFactory.ICEBERG_CATALOG_TYPE)
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Catalog type, the optional types are: custom, hadoop, hive.");
+
+ public static final ConfigOption<String> CATALOG_DATABASE =
+ ConfigOptions.key("catalog-database")
+ .stringType()
+ .defaultValue(FlinkCatalogFactory.DEFAULT_DATABASE_NAME)
+ .withDescription("Database name managed in the iceberg catalog.");
+
+ public static final ConfigOption<String> CATALOG_TABLE =
+ ConfigOptions.key("catalog-table")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Table name managed in the underlying iceberg catalog and database.");
+
+ public static final ConfigOption<Boolean> USE_DYNAMIC_ICEBERG_SINK =
+ ConfigOptions.key("use-dynamic-iceberg-sink")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether to use dynamic iceberg sink for routing data to multiple tables. "
+ + "When enabled, a single sink instance can dynamically route records to different "
+ + "Iceberg tables based on the logic defined in the DynamicRecordGenerator implementation. "
+ + "Requires 'dynamic-record-generator-impl' to be specified. "
+ + "Default is false (uses standard static sink behavior).");
+
+ public static final ConfigOption<String> DYNAMIC_RECORD_GENERATOR_IMPL =
+ ConfigOptions.key("dynamic-record-generator-impl")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Implementation of DynamicTableRecordGenerator class when use-dynamic-iceberg-sink is enabled.");
+
+ public static final String SRC_CATALOG_PROPS_KEY = "src-catalog";
+ public static final String CONNECTOR_PROPS_KEY = "connector";
+ public static final String LOCATION_KEY = "location";
+
+ static String toJson(String catalogName, String catalogDb, String catalogTable) {
+ return JsonUtil.generate(
+ gen -> {
+ gen.writeStartObject();
+ gen.writeStringField(CATALOG_NAME.key(), catalogName);
+ gen.writeStringField(CATALOG_DATABASE.key(), catalogDb);
+ gen.writeStringField(CATALOG_TABLE.key(), catalogTable);
+ gen.writeEndObject();
+ },
+ false);
+ }
+
+ static FlinkCreateTableOptions fromJson(String createTableOptions) {
+ return JsonUtil.parse(
+ createTableOptions,
+ node -> {
+ String catalogName = JsonUtil.getString(CATALOG_NAME.key(), node);
+ String catalogDb = JsonUtil.getString(CATALOG_DATABASE.key(), node);
+ String catalogTable = JsonUtil.getString(CATALOG_TABLE.key(), node);
+
+ return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable);
+ });
+ }
+
+ String catalogName() {
+ return catalogName;
+ }
+
+ String catalogDb() {
+ return catalogDb;
+ }
+
+ String catalogTable() {
+ return catalogTable;
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
new file mode 100644
index 0000000000..6306ee7a0a
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.table.catalog.CatalogDatabaseImpl;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.flink.source.IcebergTableSource;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+
+public class FlinkDynamicTableFactory
+ implements DynamicTableSinkFactory, DynamicTableSourceFactory {
+ static final String FACTORY_IDENTIFIER = "iceberg";
+ private final FlinkCatalog catalog;
+
+ public FlinkDynamicTableFactory() {
+ this.catalog = null;
+ }
+
+ public FlinkDynamicTableFactory(FlinkCatalog catalog) {
+ this.catalog = catalog;
+ }
+
+ @Override
+ public DynamicTableSource createDynamicTableSource(Context context) {
+ ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
+ ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable();
+ Map<String, String> tableProps = resolvedCatalogTable.getOptions();
+ ResolvedSchema resolvedSchema =
+ ResolvedSchema.of(
+ resolvedCatalogTable.getResolvedSchema().getColumns().stream()
+ .filter(Column::isPhysical)
+ .collect(Collectors.toList()));
+
+ TableLoader tableLoader;
+ if (catalog != null) {
+ tableLoader = createTableLoader(catalog, objectIdentifier.toObjectPath());
+ } else {
+ tableLoader =
+ createTableLoader(
+ resolvedCatalogTable,
+ tableProps,
+ objectIdentifier.getDatabaseName(),
+ objectIdentifier.getObjectName());
+ }
+
+ return new IcebergTableSource(
+ tableLoader, resolvedSchema, tableProps, context.getConfiguration());
+ }
+
+ @Override
+ public DynamicTableSink createDynamicTableSink(Context context) {
+ ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
+ ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable();
+ Map<String, String> writeProps = resolvedCatalogTable.getOptions();
+
+ Configuration flinkConf = new Configuration();
+ writeProps.forEach(flinkConf::setString);
+
+ boolean useDynamicSink = flinkConf.get(FlinkCreateTableOptions.USE_DYNAMIC_ICEBERG_SINK);
+
+ if (useDynamicSink) {
+ return getIcebergTableSinkWithDynamicSinkProps(context, flinkConf, writeProps);
+ } else {
+ TableLoader tableLoader;
+ if (catalog != null) {
+ tableLoader = createTableLoader(catalog, objectIdentifier.toObjectPath());
+ } else {
+ tableLoader =
+ createTableLoader(
+ resolvedCatalogTable,
+ writeProps,
+ objectIdentifier.getDatabaseName(),
+ objectIdentifier.getObjectName());
+ }
+
+ return new IcebergTableSink(
+ tableLoader,
+ resolvedCatalogTable.getResolvedSchema(),
+ context.getConfiguration(),
+ writeProps);
+ }
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ Set<ConfigOption<?>> options = Sets.newHashSet();
+ options.add(FlinkCreateTableOptions.CATALOG_TYPE);
+ options.add(FlinkCreateTableOptions.CATALOG_NAME);
+ return options;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ Set<ConfigOption<?>> options = Sets.newHashSet();
+ options.add(FlinkCreateTableOptions.CATALOG_DATABASE);
+ options.add(FlinkCreateTableOptions.CATALOG_TABLE);
+ options.add(FlinkCreateTableOptions.USE_DYNAMIC_ICEBERG_SINK);
+ options.add(FlinkCreateTableOptions.DYNAMIC_RECORD_GENERATOR_IMPL);
+ return options;
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return FACTORY_IDENTIFIER;
+ }
+
+ private IcebergTableSink getIcebergTableSinkWithDynamicSinkProps(
+ Context context, Configuration flinkConf, Map<String, String> writeProps) {
+ String dynamicRecordGeneratorImpl =
+ flinkConf.get(FlinkCreateTableOptions.DYNAMIC_RECORD_GENERATOR_IMPL);
+ Preconditions.checkArgument(
+ dynamicRecordGeneratorImpl != null,
+ "Invalid dynamic record generator value: null. %s must be specified when use-dynamic-iceberg-sink is true.",
+ FlinkCreateTableOptions.DYNAMIC_RECORD_GENERATOR_IMPL.key());
+
+ CatalogLoader catalogLoader;
+ if (catalog != null) {
+ catalogLoader = catalog.getCatalogLoader();
+ } else {
+ FlinkCatalog flinkCatalog =
+ createCatalogLoader(writeProps, flinkConf.get(FlinkCreateTableOptions.CATALOG_NAME));
+ catalogLoader = flinkCatalog.getCatalogLoader();
+ }
+
+ ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable();
+
+ return new IcebergTableSink(
+ catalogLoader,
+ dynamicRecordGeneratorImpl,
+ resolvedCatalogTable.getResolvedSchema(),
+ context.getConfiguration(),
+ writeProps);
+ }
+
+ private static FlinkCatalog createCatalogLoader(
+ Map<String, String> tableProps, String catalogName) {
+ Preconditions.checkArgument(
+ catalogName != null,
+ "Invalid catalog name: null. Set %s create table option.",
+ FlinkCreateTableOptions.CATALOG_NAME);
+
+ FlinkCatalogFactory factory = new FlinkCatalogFactory();
+ return (FlinkCatalog)
+ factory.createCatalog(catalogName, tableProps, FlinkCatalogFactory.clusterHadoopConf());
+ }
+
+ private static TableLoader createTableLoader(
+ ResolvedCatalogTable resolvedCatalogTable,
+ Map<String, String> tableProps,
+ String databaseName,
+ String tableName) {
+ Configuration flinkConf = new Configuration();
+
+ Map<String, String> mergedProps = mergeSrcCatalogProps(tableProps);
+
+ mergedProps.forEach(flinkConf::setString);
+
+ String catalogName = flinkConf.get(FlinkCreateTableOptions.CATALOG_NAME);
+
+ String catalogDatabase = flinkConf.get(FlinkCreateTableOptions.CATALOG_DATABASE, databaseName);
+ Preconditions.checkArgument(
+ catalogDatabase != null,
+ "Invalid database name: null. Set %s create table option or specify fully qualified table name.",
+ FlinkCreateTableOptions.CATALOG_DATABASE);
+
+ String catalogTable = flinkConf.get(FlinkCreateTableOptions.CATALOG_TABLE, tableName);
+
+ FlinkCatalog flinkCatalog = createCatalogLoader(mergedProps, catalogName);
+ ObjectPath objectPath = new ObjectPath(catalogDatabase, catalogTable);
+
+ // Create database if not exists in the external catalog.
+ if (!flinkCatalog.databaseExists(catalogDatabase)) {
+ try {
+ flinkCatalog.createDatabase(
+ catalogDatabase, new CatalogDatabaseImpl(Maps.newHashMap(), null), true);
+ } catch (DatabaseAlreadyExistException e) {
+ throw new AlreadyExistsException(
+ e,
+ "Database %s already exists in the iceberg catalog %s.",
+ catalogName,
+ catalogDatabase);
+ }
+ }
+
+ // Create table if not exists in the external catalog.
+ if (!flinkCatalog.tableExists(objectPath)) {
+ try {
+ flinkCatalog.createIcebergTable(objectPath, resolvedCatalogTable, true);
+ } catch (TableAlreadyExistException e) {
+ throw new AlreadyExistsException(
+ e,
+ "Table %s already exists in the database %s and catalog %s",
+ catalogTable,
+ catalogDatabase,
+ catalogName);
+ }
+ }
+
+ return TableLoader.fromCatalog(
+ flinkCatalog.getCatalogLoader(), TableIdentifier.of(catalogDatabase, catalogTable));
+ }
+
+ /**
+ * Merges source catalog properties (catalog name, database, table) with connector properties.
+ * Source catalog name, database, table are serialized as json in FlinkCatalog#getTable to be able
+ * to isolate them from iceberg table props, Here, we flatten and merge them back.
+ *
+ * @param tableProps the existing table properties
+ * @return a map of merged properties, defaulting to source catalog name, database and table
+ * unless overridden.
+ */
+ private static Map<String, String> mergeSrcCatalogProps(Map<String, String> tableProps) {
+ String srcCatalogProps = tableProps.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY);
+ if (srcCatalogProps != null) {
+ Map<String, String> mergedProps = Maps.newHashMap();
+ FlinkCreateTableOptions createTableOptions =
+ FlinkCreateTableOptions.fromJson(srcCatalogProps);
+
+ mergedProps.put(FlinkCreateTableOptions.CATALOG_NAME.key(), createTableOptions.catalogName());
+ mergedProps.put(
+ FlinkCreateTableOptions.CATALOG_DATABASE.key(), createTableOptions.catalogDb());
+ mergedProps.put(
+ FlinkCreateTableOptions.CATALOG_TABLE.key(), createTableOptions.catalogTable());
+
+ tableProps.forEach(
+ (k, v) -> {
+ if (!FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY.equals(k)) {
+ mergedProps.put(k, v);
+ }
+ });
+
+ return Collections.unmodifiableMap(mergedProps);
+ }
+
+ return tableProps;
+ }
+
+ private static TableLoader createTableLoader(FlinkCatalog catalog, ObjectPath objectPath) {
+ return TableLoader.fromCatalog(catalog.getCatalogLoader(), catalog.toIdentifier(objectPath));
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java
new file mode 100644
index 0000000000..f35bb577fb
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.iceberg.EnvironmentContext;
+import org.apache.iceberg.flink.util.FlinkPackage;
+
+class FlinkEnvironmentContext {
+ private FlinkEnvironmentContext() {}
+
+ public static void init() {
+ EnvironmentContext.put(EnvironmentContext.ENGINE_NAME, "flink");
+ EnvironmentContext.put(EnvironmentContext.ENGINE_VERSION, FlinkPackage.version());
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java
new file mode 100644
index 0000000000..f2244d5137
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiFunction;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.flink.table.expressions.CallExpression;
+import org.apache.flink.table.expressions.FieldReferenceExpression;
+import org.apache.flink.table.expressions.ResolvedExpression;
+import org.apache.flink.table.expressions.ValueLiteralExpression;
+import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
+import org.apache.flink.table.functions.FunctionDefinition;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expression.Operation;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.util.DateTimeUtil;
+import org.apache.iceberg.util.NaNUtil;
+
+public class FlinkFilters {
+ private FlinkFilters() {}
+
+ private static final Pattern STARTS_WITH_PATTERN = Pattern.compile("([^%]+)%");
+
+ private static final Map<FunctionDefinition, Operation> FILTERS =
+ ImmutableMap.<FunctionDefinition, Operation>builder()
+ .put(BuiltInFunctionDefinitions.EQUALS, Operation.EQ)
+ .put(BuiltInFunctionDefinitions.NOT_EQUALS, Operation.NOT_EQ)
+ .put(BuiltInFunctionDefinitions.GREATER_THAN, Operation.GT)
+ .put(BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL, Operation.GT_EQ)
+ .put(BuiltInFunctionDefinitions.LESS_THAN, Operation.LT)
+ .put(BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL, Operation.LT_EQ)
+ .put(BuiltInFunctionDefinitions.IS_NULL, Operation.IS_NULL)
+ .put(BuiltInFunctionDefinitions.IS_NOT_NULL, Operation.NOT_NULL)
+ .put(BuiltInFunctionDefinitions.AND, Operation.AND)
+ .put(BuiltInFunctionDefinitions.OR, Operation.OR)
+ .put(BuiltInFunctionDefinitions.NOT, Operation.NOT)
+ .put(BuiltInFunctionDefinitions.LIKE, Operation.STARTS_WITH)
+ .buildOrThrow();
+
+ /**
+ * Convert flink expression to iceberg expression.
+ *
+ * <p>the BETWEEN, NOT_BETWEEN, IN expression will be converted by flink automatically. the
+ * BETWEEN will be converted to (GT_EQ AND LT_EQ), the NOT_BETWEEN will be converted to (LT_EQ OR
+ * GT_EQ), the IN will be converted to OR, so we do not add the conversion here
+ *
+ * @param flinkExpression the flink expression
+ * @return the iceberg expression
+ */
+ public static Optional<Expression> convert(
+ org.apache.flink.table.expressions.Expression flinkExpression) {
+ if (!(flinkExpression instanceof CallExpression)) {
+ return Optional.empty();
+ }
+
+ CallExpression call = (CallExpression) flinkExpression;
+ Operation op = FILTERS.get(call.getFunctionDefinition());
+ if (op != null) {
+ switch (op) {
+ case IS_NULL:
+ return onlyChildAs(call, FieldReferenceExpression.class)
+ .map(FieldReferenceExpression::getName)
+ .map(Expressions::isNull);
+
+ case NOT_NULL:
+ return onlyChildAs(call, FieldReferenceExpression.class)
+ .map(FieldReferenceExpression::getName)
+ .map(Expressions::notNull);
+
+ case LT:
+ return convertFieldAndLiteral(Expressions::lessThan, Expressions::greaterThan, call);
+
+ case LT_EQ:
+ return convertFieldAndLiteral(
+ Expressions::lessThanOrEqual, Expressions::greaterThanOrEqual, call);
+
+ case GT:
+ return convertFieldAndLiteral(Expressions::greaterThan, Expressions::lessThan, call);
+
+ case GT_EQ:
+ return convertFieldAndLiteral(
+ Expressions::greaterThanOrEqual, Expressions::lessThanOrEqual, call);
+
+ case EQ:
+ return convertFieldAndLiteral(
+ (ref, lit) -> {
+ if (NaNUtil.isNaN(lit)) {
+ return Expressions.isNaN(ref);
+ } else {
+ return Expressions.equal(ref, lit);
+ }
+ },
+ call);
+
+ case NOT_EQ:
+ return convertFieldAndLiteral(
+ (ref, lit) -> {
+ if (NaNUtil.isNaN(lit)) {
+ return Expressions.notNaN(ref);
+ } else {
+ return Expressions.notEqual(ref, lit);
+ }
+ },
+ call);
+
+ case NOT:
+ return onlyChildAs(call, CallExpression.class)
+ .flatMap(FlinkFilters::convert)
+ .map(Expressions::not);
+
+ case AND:
+ return convertLogicExpression(Expressions::and, call);
+
+ case OR:
+ return convertLogicExpression(Expressions::or, call);
+
+ case STARTS_WITH:
+ return convertLike(call);
+ }
+ }
+
+ return Optional.empty();
+ }
+
+ private static <T extends ResolvedExpression> Optional<T> onlyChildAs(
+ CallExpression call, Class<T> expectedChildClass) {
+ List<ResolvedExpression> children = call.getResolvedChildren();
+ if (children.size() != 1) {
+ return Optional.empty();
+ }
+
+ ResolvedExpression child = children.get(0);
+ if (!expectedChildClass.isInstance(child)) {
+ return Optional.empty();
+ }
+
+ return Optional.of(expectedChildClass.cast(child));
+ }
+
+ private static Optional<Expression> convertLike(CallExpression call) {
+ List<ResolvedExpression> args = call.getResolvedChildren();
+ if (args.size() != 2) {
+ return Optional.empty();
+ }
+
+ org.apache.flink.table.expressions.Expression left = args.get(0);
+ org.apache.flink.table.expressions.Expression right = args.get(1);
+
+ if (left instanceof FieldReferenceExpression && right instanceof ValueLiteralExpression) {
+ String name = ((FieldReferenceExpression) left).getName();
+ return convertLiteral((ValueLiteralExpression) right)
+ .flatMap(
+ lit -> {
+ if (lit instanceof String) {
+ String pattern = (String) lit;
+ Matcher matcher = STARTS_WITH_PATTERN.matcher(pattern);
+ // exclude special char of LIKE
+ // '_' is the wildcard of the SQL LIKE
+ if (!pattern.contains("_") && matcher.matches()) {
+ return Optional.of(Expressions.startsWith(name, matcher.group(1)));
+ }
+ }
+
+ return Optional.empty();
+ });
+ }
+
+ return Optional.empty();
+ }
+
+ private static Optional<Expression> convertLogicExpression(
+ BiFunction<Expression, Expression, Expression> function, CallExpression call) {
+ List<ResolvedExpression> args = call.getResolvedChildren();
+ if (args == null || args.size() != 2) {
+ return Optional.empty();
+ }
+
+ Optional<Expression> left = convert(args.get(0));
+ Optional<Expression> right = convert(args.get(1));
+ if (left.isPresent() && right.isPresent()) {
+ return Optional.of(function.apply(left.get(), right.get()));
+ }
+
+ return Optional.empty();
+ }
+
+ private static Optional<Object> convertLiteral(ValueLiteralExpression expression) {
+ Optional<?> value =
+ expression.getValueAs(
+ expression.getOutputDataType().getLogicalType().getDefaultConversion());
+ return value.map(
+ o -> {
+ if (o instanceof LocalDateTime) {
+ return DateTimeUtil.microsFromTimestamp((LocalDateTime) o);
+ } else if (o instanceof Instant) {
+ return DateTimeUtil.microsFromInstant((Instant) o);
+ } else if (o instanceof LocalTime) {
+ return DateTimeUtil.microsFromTime((LocalTime) o);
+ } else if (o instanceof LocalDate) {
+ return DateTimeUtil.daysFromDate((LocalDate) o);
+ }
+
+ return o;
+ });
+ }
+
+ private static Optional<Expression> convertFieldAndLiteral(
+ BiFunction<String, Object, Expression> expr, CallExpression call) {
+ return convertFieldAndLiteral(expr, expr, call);
+ }
+
+ private static Optional<Expression> convertFieldAndLiteral(
+ BiFunction<String, Object, Expression> convertLR,
+ BiFunction<String, Object, Expression> convertRL,
+ CallExpression call) {
+ List<ResolvedExpression> args = call.getResolvedChildren();
+ if (args.size() != 2) {
+ return Optional.empty();
+ }
+
+ org.apache.flink.table.expressions.Expression left = args.get(0);
+ org.apache.flink.table.expressions.Expression right = args.get(1);
+
+ if (left instanceof FieldReferenceExpression && right instanceof ValueLiteralExpression) {
+ String name = ((FieldReferenceExpression) left).getName();
+ Optional<Object> lit = convertLiteral((ValueLiteralExpression) right);
+ if (lit.isPresent()) {
+ return Optional.of(convertLR.apply(name, lit.get()));
+ }
+ } else if (left instanceof ValueLiteralExpression
+ && right instanceof FieldReferenceExpression) {
+ Optional<Object> lit = convertLiteral((ValueLiteralExpression) left);
+ String name = ((FieldReferenceExpression) right).getName();
+ if (lit.isPresent()) {
+ return Optional.of(convertRL.apply(name, lit.get()));
+ }
+ }
+
+ return Optional.empty();
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java
new file mode 100644
index 0000000000..767d4497ac
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.types.FixupTypes;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+/**
+ * The uuid and fixed are converted to the same Flink type. Conversion back can produce only one,
+ * which may not be correct.
+ */
+class FlinkFixupTypes extends FixupTypes {
+
+ private FlinkFixupTypes(Schema referenceSchema) {
+ super(referenceSchema);
+ }
+
+ static Schema fixup(Schema schema, Schema referenceSchema) {
+ return new Schema(
+ TypeUtil.visit(schema, new FlinkFixupTypes(referenceSchema)).asStructType().fields());
+ }
+
+ @Override
+ protected boolean fixupPrimitive(Type.PrimitiveType type, Type source) {
+ if (type instanceof Types.FixedType) {
+ int length = ((Types.FixedType) type).length();
+ return source.typeId() == Type.TypeID.UUID && length == 16;
+ }
+ return false;
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java
new file mode 100644
index 0000000000..804a956ec9
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.util.TimeUtils;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.flink.source.StreamingStartingStrategy;
+
+public class FlinkReadConf {
+
+ private final FlinkConfParser confParser;
+
+ public FlinkReadConf(
+ Table table, Map<String, String> readOptions, ReadableConfig readableConfig) {
+ this.confParser = new FlinkConfParser(table, readOptions, readableConfig);
+ }
+
+ public Long snapshotId() {
+ return confParser.longConf().option(FlinkReadOptions.SNAPSHOT_ID.key()).parseOptional();
+ }
+
+ public String tag() {
+ return confParser.stringConf().option(FlinkReadOptions.TAG.key()).parseOptional();
+ }
+
+ public String startTag() {
+ return confParser.stringConf().option(FlinkReadOptions.START_TAG.key()).parseOptional();
+ }
+
+ public String endTag() {
+ return confParser.stringConf().option(FlinkReadOptions.END_TAG.key()).parseOptional();
+ }
+
+ public String branch() {
+ return confParser.stringConf().option(FlinkReadOptions.BRANCH.key()).parseOptional();
+ }
+
+ public boolean caseSensitive() {
+ return confParser
+ .booleanConf()
+ .option(FlinkReadOptions.CASE_SENSITIVE)
+ .flinkConfig(FlinkReadOptions.CASE_SENSITIVE_OPTION)
+ .defaultValue(FlinkReadOptions.CASE_SENSITIVE_OPTION.defaultValue())
+ .parse();
+ }
+
+ public Long asOfTimestamp() {
+ return confParser.longConf().option(FlinkReadOptions.AS_OF_TIMESTAMP.key()).parseOptional();
+ }
+
+ public StreamingStartingStrategy startingStrategy() {
+ return confParser
+ .enumConfParser(StreamingStartingStrategy.class)
+ .option(FlinkReadOptions.STARTING_STRATEGY)
+ .flinkConfig(FlinkReadOptions.STARTING_STRATEGY_OPTION)
+ .defaultValue(StreamingStartingStrategy.INCREMENTAL_FROM_LATEST_SNAPSHOT)
+ .parse();
+ }
+
+ public Long startSnapshotTimestamp() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.START_SNAPSHOT_TIMESTAMP.key())
+ .parseOptional();
+ }
+
+ public Long startSnapshotId() {
+ return confParser.longConf().option(FlinkReadOptions.START_SNAPSHOT_ID.key()).parseOptional();
+ }
+
+ public Long endSnapshotId() {
+ return confParser.longConf().option(FlinkReadOptions.END_SNAPSHOT_ID.key()).parseOptional();
+ }
+
+ public long splitSize() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.SPLIT_SIZE)
+ .flinkConfig(FlinkReadOptions.SPLIT_SIZE_OPTION)
+ .tableProperty(TableProperties.SPLIT_SIZE)
+ .defaultValue(TableProperties.SPLIT_SIZE_DEFAULT)
+ .parse();
+ }
+
+ public int splitLookback() {
+ return confParser
+ .intConf()
+ .option(FlinkReadOptions.SPLIT_LOOKBACK)
+ .flinkConfig(FlinkReadOptions.SPLIT_LOOKBACK_OPTION)
+ .tableProperty(TableProperties.SPLIT_LOOKBACK)
+ .defaultValue(TableProperties.SPLIT_LOOKBACK_DEFAULT)
+ .parse();
+ }
+
+ public long splitFileOpenCost() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.SPLIT_FILE_OPEN_COST)
+ .flinkConfig(FlinkReadOptions.SPLIT_FILE_OPEN_COST_OPTION)
+ .tableProperty(TableProperties.SPLIT_OPEN_FILE_COST)
+ .defaultValue(TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT)
+ .parse();
+ }
+
+ public boolean streaming() {
+ return confParser
+ .booleanConf()
+ .option(FlinkReadOptions.STREAMING)
+ .flinkConfig(FlinkReadOptions.STREAMING_OPTION)
+ .defaultValue(FlinkReadOptions.STREAMING_OPTION.defaultValue())
+ .parse();
+ }
+
+ public Duration monitorInterval() {
+ String duration =
+ confParser
+ .stringConf()
+ .option(FlinkReadOptions.MONITOR_INTERVAL)
+ .flinkConfig(FlinkReadOptions.MONITOR_INTERVAL_OPTION)
+ .defaultValue(FlinkReadOptions.MONITOR_INTERVAL_OPTION.defaultValue())
+ .parse();
+
+ return TimeUtils.parseDuration(duration);
+ }
+
+ public boolean includeColumnStats() {
+ return confParser
+ .booleanConf()
+ .option(FlinkReadOptions.INCLUDE_COLUMN_STATS)
+ .flinkConfig(FlinkReadOptions.INCLUDE_COLUMN_STATS_OPTION)
+ .defaultValue(FlinkReadOptions.INCLUDE_COLUMN_STATS_OPTION.defaultValue())
+ .parse();
+ }
+
+ public int maxPlanningSnapshotCount() {
+ return confParser
+ .intConf()
+ .option(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT)
+ .flinkConfig(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT_OPTION)
+ .defaultValue(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT_OPTION.defaultValue())
+ .parse();
+ }
+
+ public String nameMapping() {
+ return confParser.stringConf().option(TableProperties.DEFAULT_NAME_MAPPING).parseOptional();
+ }
+
+ public long limit() {
+ return confParser
+ .longConf()
+ .option(FlinkReadOptions.LIMIT)
+ .flinkConfig(FlinkReadOptions.LIMIT_OPTION)
+ .defaultValue(FlinkReadOptions.LIMIT_OPTION.defaultValue())
+ .parse();
+ }
+
+ public int workerPoolSize() {
+ return confParser
+ .intConf()
+ .option(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.key())
+ .flinkConfig(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE)
+ .defaultValue(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.defaultValue())
+ .parse();
+ }
+
+ public int maxAllowedPlanningFailures() {
+ return confParser
+ .intConf()
+ .option(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES)
+ .flinkConfig(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES_OPTION)
+ .defaultValue(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES_OPTION.defaultValue())
+ .parse();
+ }
+
+ public String watermarkColumn() {
+ return confParser
+ .stringConf()
+ .option(FlinkReadOptions.WATERMARK_COLUMN)
+ .flinkConfig(FlinkReadOptions.WATERMARK_COLUMN_OPTION)
+ .defaultValue(FlinkReadOptions.WATERMARK_COLUMN_OPTION.defaultValue())
+ .parseOptional();
+ }
+
+ public TimeUnit watermarkColumnTimeUnit() {
+ return confParser
+ .enumConfParser(TimeUnit.class)
+ .option(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT)
+ .flinkConfig(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT_OPTION)
+ .defaultValue(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT_OPTION.defaultValue())
+ .parse();
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java
new file mode 100644
index 0000000000..0616d11349
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.iceberg.flink.source.StreamingStartingStrategy;
+
+/** Flink source read options */
+public class FlinkReadOptions {
+ private static final String PREFIX = "connector.iceberg.";
+
+ private FlinkReadOptions() {}
+
+ public static final ConfigOption<Long> SNAPSHOT_ID =
+ ConfigOptions.key("snapshot-id").longType().defaultValue(null);
+
+ public static final ConfigOption<String> TAG =
+ ConfigOptions.key("tag").stringType().defaultValue(null);
+
+ public static final ConfigOption<String> BRANCH =
+ ConfigOptions.key("branch").stringType().defaultValue(null);
+
+ public static final ConfigOption<String> START_TAG =
+ ConfigOptions.key("start-tag").stringType().defaultValue(null);
+
+ public static final ConfigOption<String> END_TAG =
+ ConfigOptions.key("end-tag").stringType().defaultValue(null);
+
+ public static final String CASE_SENSITIVE = "case-sensitive";
+ public static final ConfigOption<Boolean> CASE_SENSITIVE_OPTION =
+ ConfigOptions.key(PREFIX + CASE_SENSITIVE).booleanType().defaultValue(false);
+
+ public static final ConfigOption<Long> AS_OF_TIMESTAMP =
+ ConfigOptions.key("as-of-timestamp").longType().defaultValue(null);
+
+ public static final String STARTING_STRATEGY = "starting-strategy";
+ public static final ConfigOption<StreamingStartingStrategy> STARTING_STRATEGY_OPTION =
+ ConfigOptions.key(PREFIX + STARTING_STRATEGY)
+ .enumType(StreamingStartingStrategy.class)
+ .defaultValue(StreamingStartingStrategy.INCREMENTAL_FROM_LATEST_SNAPSHOT);
+
+ public static final ConfigOption<Long> START_SNAPSHOT_TIMESTAMP =
+ ConfigOptions.key("start-snapshot-timestamp").longType().defaultValue(null);
+
+ public static final ConfigOption<Long> START_SNAPSHOT_ID =
+ ConfigOptions.key("start-snapshot-id").longType().defaultValue(null);
+
+ public static final ConfigOption<Long> END_SNAPSHOT_ID =
+ ConfigOptions.key("end-snapshot-id").longType().defaultValue(null);
+
+ // No defaults for the split options: a declared default would mask the table property fallback
+ // in FlinkReadConf, which applies the default instead
+ public static final String SPLIT_SIZE = "split-size";
+ public static final ConfigOption<Long> SPLIT_SIZE_OPTION =
+ ConfigOptions.key(PREFIX + SPLIT_SIZE).longType().noDefaultValue();
+
+ public static final String SPLIT_LOOKBACK = "split-lookback";
+ public static final ConfigOption<Integer> SPLIT_LOOKBACK_OPTION =
+ ConfigOptions.key(PREFIX + SPLIT_LOOKBACK).intType().noDefaultValue();
+
+ public static final String SPLIT_FILE_OPEN_COST = "split-file-open-cost";
+ public static final ConfigOption<Long> SPLIT_FILE_OPEN_COST_OPTION =
+ ConfigOptions.key(PREFIX + SPLIT_FILE_OPEN_COST).longType().noDefaultValue();
+
+ public static final String STREAMING = "streaming";
+ public static final ConfigOption<Boolean> STREAMING_OPTION =
+ ConfigOptions.key(PREFIX + STREAMING).booleanType().defaultValue(false);
+
+ public static final String MONITOR_INTERVAL = "monitor-interval";
+ public static final ConfigOption<String> MONITOR_INTERVAL_OPTION =
+ ConfigOptions.key(PREFIX + MONITOR_INTERVAL).stringType().defaultValue("60s");
+
+ public static final String INCLUDE_COLUMN_STATS = "include-column-stats";
+ public static final ConfigOption<Boolean> INCLUDE_COLUMN_STATS_OPTION =
+ ConfigOptions.key(PREFIX + INCLUDE_COLUMN_STATS).booleanType().defaultValue(false);
+
+ public static final String MAX_PLANNING_SNAPSHOT_COUNT = "max-planning-snapshot-count";
+ public static final ConfigOption<Integer> MAX_PLANNING_SNAPSHOT_COUNT_OPTION =
+ ConfigOptions.key(PREFIX + MAX_PLANNING_SNAPSHOT_COUNT)
+ .intType()
+ .defaultValue(Integer.MAX_VALUE);
+
+ public static final String LIMIT = "limit";
+ public static final ConfigOption<Long> LIMIT_OPTION =
+ ConfigOptions.key(PREFIX + LIMIT).longType().defaultValue(-1L);
+
+ public static final String MAX_ALLOWED_PLANNING_FAILURES = "max-allowed-planning-failures";
+ public static final ConfigOption<Integer> MAX_ALLOWED_PLANNING_FAILURES_OPTION =
+ ConfigOptions.key(PREFIX + MAX_ALLOWED_PLANNING_FAILURES).intType().defaultValue(3);
+
+ public static final String WATERMARK_COLUMN = "watermark-column";
+ public static final ConfigOption<String> WATERMARK_COLUMN_OPTION =
+ ConfigOptions.key(PREFIX + WATERMARK_COLUMN).stringType().noDefaultValue();
+
+ public static final String WATERMARK_COLUMN_TIME_UNIT = "watermark-column-time-unit";
+ public static final ConfigOption<TimeUnit> WATERMARK_COLUMN_TIME_UNIT_OPTION =
+ ConfigOptions.key(PREFIX + WATERMARK_COLUMN_TIME_UNIT)
+ .enumType(TimeUnit.class)
+ .defaultValue(TimeUnit.MICROSECONDS);
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkRowData.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkRowData.java
new file mode 100644
index 0000000000..06c1635312
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkRowData.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.NullType;
+
+public class FlinkRowData {
+
+ private FlinkRowData() {}
+
+ public static RowData.FieldGetter createFieldGetter(LogicalType fieldType, int fieldPos) {
+ if (fieldType instanceof NullType) {
+ return rowData -> null;
+ }
+
+ RowData.FieldGetter flinkFieldGetter = RowData.createFieldGetter(fieldType, fieldPos);
+ return rowData -> {
+ // Be sure to check for null values, even if the field is required. Flink
+ // RowData.createFieldGetter(..) does not null-check optional / nullable types. Without this
+ // explicit null check, the null flag of BinaryRowData will be ignored and random bytes will
+ // be parsed as actual values. This will produce incorrect writes instead of failing with a
+ // NullPointerException. See https://issues.apache.org/jira/browse/FLINK-37245
+ if (!fieldType.isNullable() && rowData.isNullAt(fieldPos)) {
+ return null;
+ }
+ return flinkFieldGetter.getFieldOrNull(rowData);
+ };
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
new file mode 100644
index 0000000000..7f55d4b07b
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.catalog.UniqueConstraint;
+import org.apache.flink.table.legacy.api.TableSchema;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.utils.TypeConversions;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+/**
+ * Converter between Flink types and Iceberg type. The conversion is not a 1:1 mapping that not
+ * allows back-and-forth conversion. So some information might get lost during the back-and-forth
+ * conversion.
+ *
+ * <p>This inconsistent types:
+ *
+ * <ul>
+ * <li>map Iceberg UUID type to Flink BinaryType(16)
+ * <li>map Flink VarCharType(_) and CharType(_) to Iceberg String type
+ * <li>map Flink VarBinaryType(_) to Iceberg Binary type
+ * <li>map Flink TimeType(_) to Iceberg Time type (microseconds)
+ * <li>map Flink TimestampType(_) to Iceberg Timestamp without zone type (microseconds)
+ * <li>map Flink LocalZonedTimestampType(_) to Iceberg Timestamp with zone type (microseconds)
+ * <li>map Flink MultiSetType to Iceberg Map type(element, int)
+ * </ul>
+ *
+ * <p>
+ */
+public class FlinkSchemaUtil {
+
+ private FlinkSchemaUtil() {}
+
+ /**
+ * @deprecated will be removed in 2.0.0; use {@link #convert(ResolvedSchema)} instead.
+ */
+ @Deprecated
+ public static Schema convert(TableSchema schema) {
+ LogicalType schemaType = schema.toRowDataType().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be row type.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ if (schema.getPrimaryKey().isPresent()) {
+ return freshIdentifierFieldIds(icebergSchema, schema.getPrimaryKey().get().getColumns());
+ } else {
+ return icebergSchema;
+ }
+ }
+
+ /** Convert the flink table schema to apache iceberg schema with column comment. */
+ public static Schema convert(ResolvedSchema flinkSchema) {
+ List<Column> tableColumns = flinkSchema.getColumns();
+ // copy from org.apache.flink.table.api.Schema#toRowDataType
+ DataTypes.Field[] fields =
+ tableColumns.stream()
+ .map(
+ column -> {
+ if (column.getComment().isPresent()) {
+ return DataTypes.FIELD(
+ column.getName(), column.getDataType(), column.getComment().get());
+ } else {
+ return DataTypes.FIELD(column.getName(), column.getDataType());
+ }
+ })
+ .toArray(DataTypes.Field[]::new);
+
+ LogicalType schemaType = DataTypes.ROW(fields).notNull().getLogicalType();
+ Preconditions.checkArgument(
+ schemaType instanceof RowType, "Schema logical type should be row type.");
+
+ RowType root = (RowType) schemaType;
+ Type converted = root.accept(new FlinkTypeToType(root));
+ Schema icebergSchema = new Schema(converted.asStructType().fields());
+ return flinkSchema
+ .getPrimaryKey()
+ .map(pk -> freshIdentifierFieldIds(icebergSchema, pk.getColumns()))
+ .orElse(icebergSchema);
+ }
+
+ private static Schema freshIdentifierFieldIds(Schema icebergSchema, List<String> primaryKeys) {
+ // Locate the identifier field id list.
+ Set<Integer> identifierFieldIds = Sets.newHashSet();
+ for (String primaryKey : primaryKeys) {
+ Types.NestedField field = icebergSchema.findField(primaryKey);
+ Preconditions.checkNotNull(
+ field,
+ "Cannot find field ID for the primary key column %s in schema %s",
+ primaryKey,
+ icebergSchema);
+ identifierFieldIds.add(field.fieldId());
+ }
+ return new Schema(
+ icebergSchema.schemaId(), icebergSchema.asStruct().fields(), identifierFieldIds);
+ }
+
+ /**
+ * Convert a Flink {@link TableSchema} to a {@link Schema} based on the given schema.
+ *
+ * <p>This conversion does not assign new ids; it uses ids from the base schema.
+ *
+ * <p>Data types, field order, and nullability will match the Flink type. This conversion may
+ * return a schema that is not compatible with base schema.
+ *
+ * @param baseSchema a Schema on which conversion is based
+ * @param flinkSchema a Flink TableSchema
+ * @return the equivalent Schema
+ * @throws IllegalArgumentException if the type cannot be converted or there are missing ids
+ * @deprecated since 1.10.0, will be removed in 2.0.0. Use {@link #convert(Schema,
+ * ResolvedSchema)} instead.
+ */
+ @Deprecated
+ public static Schema convert(Schema baseSchema, TableSchema flinkSchema) {
+ // convert to a type with fresh ids
+ Types.StructType struct = convert(flinkSchema).asStruct();
+ // reassign ids to match the base schema
+ Schema schema = TypeUtil.reassignIds(new Schema(struct.fields()), baseSchema);
+ // reassign doc to match the base schema
+ schema = TypeUtil.reassignDoc(schema, baseSchema);
+
+ // fix types that can't be represented in Flink (UUID)
+ Schema fixedSchema = FlinkFixupTypes.fixup(schema, baseSchema);
+ if (flinkSchema.getPrimaryKey().isPresent()) {
+ return freshIdentifierFieldIds(fixedSchema, flinkSchema.getPrimaryKey().get().getColumns());
+ } else {
+ return fixedSchema;
+ }
+ }
+
+ /**
+ * Convert a Flink {@link ResolvedSchema} to a {@link Schema} based on the given schema.
+ *
+ * <p>This conversion does not assign new ids; it uses ids from the base schema.
+ *
+ * <p>Data types, field order, and nullability will match the Flink type. This conversion may
+ * return a schema that is not compatible with base schema.
+ *
+ * @param baseSchema a Schema on which conversion is based
+ * @param flinkSchema a Flink ResolvedSchema
+ * @return the equivalent Schema
+ * @throws IllegalArgumentException if the type cannot be converted or there are missing ids
+ */
+ public static Schema convert(Schema baseSchema, ResolvedSchema flinkSchema) {
+ // convert to a type with fresh ids
+ Types.StructType struct = convert(flinkSchema).asStruct();
+ // reassign ids to match the base schema
+ Schema schema = TypeUtil.reassignIds(new Schema(struct.fields()), baseSchema);
+ // reassign doc to match the base schema
+ schema = TypeUtil.reassignDoc(schema, baseSchema);
+
+ // fix types that can't be represented in Flink (UUID)
+ Schema fixedSchema = FlinkFixupTypes.fixup(schema, baseSchema);
+ return flinkSchema
+ .getPrimaryKey()
+ .map(pk -> freshIdentifierFieldIds(fixedSchema, pk.getColumns()))
+ .orElse(fixedSchema);
+ }
+
+ /**
+ * Convert a {@link Schema} to a {@link RowType Flink type}.
+ *
+ * @param schema a Schema
+ * @return the equivalent Flink type
+ * @throws IllegalArgumentException if the type cannot be converted to Flink
+ */
+ public static RowType convert(Schema schema) {
+ return (RowType) TypeUtil.visit(schema, new TypeToFlinkType());
+ }
+
+ /**
+ * Convert a {@link Type} to a {@link LogicalType Flink type}.
+ *
+ * @param type a Type
+ * @return the equivalent Flink type
+ * @throws IllegalArgumentException if the type cannot be converted to Flink
+ */
+ public static LogicalType convert(Type type) {
+ return TypeUtil.visit(type, new TypeToFlinkType());
+ }
+
+ /**
+ * Convert a {@link LogicalType Flink type} to a {@link Type}.
+ *
+ * @param flinkType a FlinkType
+ * @return the equivalent Iceberg type
+ */
+ public static Type convert(LogicalType flinkType) {
+ return flinkType.accept(new FlinkTypeToType());
+ }
+
+ /**
+ * Convert a {@link RowType} to a {@link TableSchema}.
+ *
+ * @param rowType a RowType
+ * @return Flink TableSchema
+ * @deprecated since 1.10.0, will be removed in 2.0.0. Use {@link #toResolvedSchema(RowType)}
+ * instead
+ */
+ @Deprecated
+ public static TableSchema toSchema(RowType rowType) {
+ TableSchema.Builder builder = TableSchema.builder();
+ for (RowType.RowField field : rowType.getFields()) {
+ builder.field(field.getName(), TypeConversions.fromLogicalToDataType(field.getType()));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Convert a {@link RowType} to a {@link ResolvedSchema}.
+ *
+ * @param rowType a RowType
+ * @return Flink ResolvedSchema
+ */
+ public static ResolvedSchema toResolvedSchema(RowType rowType) {
+ List<Column> columns = Lists.newArrayListWithExpectedSize(rowType.getFieldCount());
+ for (RowType.RowField field : rowType.getFields()) {
+ columns.add(
+ Column.physical(field.getName(), TypeConversions.fromLogicalToDataType(field.getType())));
+ }
+
+ return ResolvedSchema.of(columns);
+ }
+
+ /**
+ * Convert a {@link Schema} to a {@link TableSchema}.
+ *
+ * @param schema iceberg schema to convert.
+ * @return Flink TableSchema.
+ * @deprecated since 1.10.0, will be removed in 2.0.0. Use {@link #toResolvedSchema(Schema)}
+ * instead
+ */
+ @Deprecated
+ public static TableSchema toSchema(Schema schema) {
+ TableSchema.Builder builder = TableSchema.builder();
+
+ // Add columns.
+ for (RowType.RowField field : convert(schema).getFields()) {
+ builder.field(field.getName(), TypeConversions.fromLogicalToDataType(field.getType()));
+ }
+
+ // Add primary key.
+ Set<Integer> identifierFieldIds = schema.identifierFieldIds();
+ if (!identifierFieldIds.isEmpty()) {
+ List<String> columns = Lists.newArrayListWithExpectedSize(identifierFieldIds.size());
+ for (Integer identifierFieldId : identifierFieldIds) {
+ String columnName = schema.findColumnName(identifierFieldId);
+ Preconditions.checkNotNull(
+ columnName, "Cannot find field with id %s in schema %s", identifierFieldId, schema);
+
+ columns.add(columnName);
+ }
+ builder.primaryKey(columns.toArray(new String[0]));
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Convert a {@link Schema} to a {@link ResolvedSchema}.
+ *
+ * @param schema iceberg schema to convert.
+ * @return Flink ResolvedSchema.
+ */
+ public static ResolvedSchema toResolvedSchema(Schema schema) {
+ RowType rowType = convert(schema);
+ List<Column> columns = Lists.newArrayListWithExpectedSize(rowType.getFieldCount());
+
+ // Add columns.
+ for (RowType.RowField field : rowType.getFields()) {
+ columns.add(
+ Column.physical(field.getName(), TypeConversions.fromLogicalToDataType(field.getType())));
+ }
+
+ // Add primary key.
+ Set<Integer> identifierFieldIds = schema.identifierFieldIds();
+ UniqueConstraint uniqueConstraint = null;
+ if (!identifierFieldIds.isEmpty()) {
+ List<String> primaryKeyColumns =
+ Lists.newArrayListWithExpectedSize(identifierFieldIds.size());
+ for (Integer identifierFieldId : identifierFieldIds) {
+ String columnName = schema.findColumnName(identifierFieldId);
+ Preconditions.checkNotNull(
+ columnName, "Cannot find field with id %s in schema %s", identifierFieldId, schema);
+
+ primaryKeyColumns.add(columnName);
+ }
+
+ uniqueConstraint =
+ UniqueConstraint.primaryKey(UUID.randomUUID().toString(), primaryKeyColumns);
+
+ validatePrimaryKey(uniqueConstraint, columns);
+ }
+
+ return new ResolvedSchema(columns, Collections.emptyList(), uniqueConstraint);
+ }
+
+ /**
+ * Copied from
+ * org.apache.flink.table.catalog.DefaultSchemaResolver#validatePrimaryKey(org.apache.flink.table.catalog.UniqueConstraint,
+ * java.util.List)
+ */
+ private static void validatePrimaryKey(UniqueConstraint primaryKey, List<Column> columns) {
+ final Map<String, Column> columnsByNameLookup =
+ columns.stream().collect(Collectors.toMap(Column::getName, Function.identity()));
+
+ final Set<String> duplicateColumns =
+ primaryKey.getColumns().stream()
+ .filter(name -> Collections.frequency(primaryKey.getColumns(), name) > 1)
+ .collect(Collectors.toSet());
+
+ if (!duplicateColumns.isEmpty()) {
+ throw new ValidationException(
+ String.format(
+ "Invalid primary key '%s'. A primary key must not contain duplicate columns. Found: %s",
+ primaryKey.getName(), duplicateColumns));
+ }
+
+ for (String columnName : primaryKey.getColumns()) {
+ Column column = columnsByNameLookup.get(columnName);
+ if (column == null) {
+ throw new ValidationException(
+ String.format(
+ "Invalid primary key '%s'. Column '%s' does not exist.",
+ primaryKey.getName(), columnName));
+ }
+
+ if (!column.isPhysical()) {
+ throw new ValidationException(
+ String.format(
+ "Invalid primary key '%s'. Column '%s' is not a physical column.",
+ primaryKey.getName(), columnName));
+ }
+
+ final LogicalType columnType = column.getDataType().getLogicalType();
+ if (columnType.isNullable()) {
+ throw new ValidationException(
+ String.format(
+ "Invalid primary key '%s'. Column '%s' is nullable.",
+ primaryKey.getName(), columnName));
+ }
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java
new file mode 100644
index 0000000000..5fbd84909d
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.expressions.Evaluator;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.types.Types;
+
+public class FlinkSourceFilter implements FilterFunction<RowData> {
+
+ private final RowType rowType;
+ private final Evaluator evaluator;
+ private final Types.StructType struct;
+ private volatile RowDataWrapper wrapper;
+
+ public FlinkSourceFilter(Schema schema, Expression expr, boolean caseSensitive) {
+ this.rowType = FlinkSchemaUtil.convert(schema);
+ this.struct = schema.asStruct();
+ this.evaluator = new Evaluator(struct, expr, caseSensitive);
+ }
+
+ @Override
+ public boolean filter(RowData value) {
+ if (wrapper == null) {
+ this.wrapper = new RowDataWrapper(rowType, struct);
+ }
+ return evaluator.eval(wrapper.wrap(value));
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java
new file mode 100644
index 0000000000..bfcd34d0b9
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.BigIntType;
+import org.apache.flink.table.types.logical.BinaryType;
+import org.apache.flink.table.types.logical.BooleanType;
+import org.apache.flink.table.types.logical.CharType;
+import org.apache.flink.table.types.logical.DateType;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.DoubleType;
+import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.MultisetType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.SmallIntType;
+import org.apache.flink.table.types.logical.TimeType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.table.types.logical.TinyIntType;
+import org.apache.flink.table.types.logical.VarBinaryType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.flink.table.types.logical.VariantType;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+class FlinkTypeToType extends FlinkTypeVisitor<Type> {
+
+ private final RowType root;
+ private int nextId;
+
+ FlinkTypeToType() {
+ this.root = null;
+ }
+
+ FlinkTypeToType(RowType root) {
+ this.root = root;
+ // the root struct's fields use the first ids
+ this.nextId = root.getFieldCount();
+ }
+
+ private int getNextId() {
+ int next = nextId;
+ nextId += 1;
+ return next;
+ }
+
+ @Override
+ public Type visit(CharType charType) {
+ return Types.StringType.get();
+ }
+
+ @Override
+ public Type visit(VarCharType varCharType) {
+ return Types.StringType.get();
+ }
+
+ @Override
+ public Type visit(BooleanType booleanType) {
+ return Types.BooleanType.get();
+ }
+
+ @Override
+ public Type visit(BinaryType binaryType) {
+ return Types.FixedType.ofLength(binaryType.getLength());
+ }
+
+ @Override
+ public Type visit(VarBinaryType varBinaryType) {
+ return Types.BinaryType.get();
+ }
+
+ @Override
+ public Type visit(DecimalType decimalType) {
+ return Types.DecimalType.of(decimalType.getPrecision(), decimalType.getScale());
+ }
+
+ @Override
+ public Type visit(TinyIntType tinyIntType) {
+ return Types.IntegerType.get();
+ }
+
+ @Override
+ public Type visit(SmallIntType smallIntType) {
+ return Types.IntegerType.get();
+ }
+
+ @Override
+ public Type visit(IntType intType) {
+ return Types.IntegerType.get();
+ }
+
+ @Override
+ public Type visit(BigIntType bigIntType) {
+ return Types.LongType.get();
+ }
+
+ @Override
+ public Type visit(FloatType floatType) {
+ return Types.FloatType.get();
+ }
+
+ @Override
+ public Type visit(DoubleType doubleType) {
+ return Types.DoubleType.get();
+ }
+
+ @Override
+ public Type visit(DateType dateType) {
+ return Types.DateType.get();
+ }
+
+ @Override
+ public Type visit(TimeType timeType) {
+ return Types.TimeType.get();
+ }
+
+ @Override
+ public Type visit(TimestampType timestampType) {
+ if (timestampType.getPrecision() > 6) {
+ return Types.TimestampNanoType.withoutZone();
+ }
+ return Types.TimestampType.withoutZone();
+ }
+
+ @Override
+ public Type visit(LocalZonedTimestampType localZonedTimestampType) {
+ if (localZonedTimestampType.getPrecision() > 6) {
+ return Types.TimestampNanoType.withZone();
+ }
+ return Types.TimestampType.withZone();
+ }
+
+ @Override
+ public Type visit(ArrayType arrayType) {
+ Type elementType = arrayType.getElementType().accept(this);
+ if (arrayType.getElementType().isNullable()) {
+ return Types.ListType.ofOptional(getNextId(), elementType);
+ } else {
+ return Types.ListType.ofRequired(getNextId(), elementType);
+ }
+ }
+
+ @Override
+ public Type visit(MultisetType multisetType) {
+ Type elementType = multisetType.getElementType().accept(this);
+ return Types.MapType.ofRequired(getNextId(), getNextId(), elementType, Types.IntegerType.get());
+ }
+
+ @Override
+ public Type visit(MapType mapType) {
+ // keys in map are not allowed to be null.
+ Type keyType = mapType.getKeyType().accept(this);
+ Type valueType = mapType.getValueType().accept(this);
+ if (mapType.getValueType().isNullable()) {
+ return Types.MapType.ofOptional(getNextId(), getNextId(), keyType, valueType);
+ } else {
+ return Types.MapType.ofRequired(getNextId(), getNextId(), keyType, valueType);
+ }
+ }
+
+ @Override
+ @SuppressWarnings("ReferenceEquality")
+ public Type visit(RowType rowType) {
+ List<Types.NestedField> newFields = Lists.newArrayListWithExpectedSize(rowType.getFieldCount());
+ boolean isRoot = root == rowType;
+
+ List<Type> types =
+ rowType.getFields().stream()
+ .map(f -> f.getType().accept(this))
+ .collect(Collectors.toList());
+
+ for (int i = 0; i < rowType.getFieldCount(); i++) {
+ int id = isRoot ? i : getNextId();
+
+ RowType.RowField field = rowType.getFields().get(i);
+ String name = field.getName();
+ String comment = field.getDescription().orElse(null);
+
+ if (field.getType().isNullable()) {
+ newFields.add(Types.NestedField.optional(id, name, types.get(i), comment));
+ } else {
+ newFields.add(Types.NestedField.required(id, name, types.get(i), comment));
+ }
+ }
+
+ return Types.StructType.of(newFields);
+ }
+
+ @Override
+ public Type visit(VariantType variantType) {
+ return Types.VariantType.get();
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java
new file mode 100644
index 0000000000..f3de241608
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import org.apache.flink.table.types.logical.DayTimeIntervalType;
+import org.apache.flink.table.types.logical.DistinctType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeVisitor;
+import org.apache.flink.table.types.logical.NullType;
+import org.apache.flink.table.types.logical.RawType;
+import org.apache.flink.table.types.logical.StructuredType;
+import org.apache.flink.table.types.logical.SymbolType;
+import org.apache.flink.table.types.logical.YearMonthIntervalType;
+import org.apache.flink.table.types.logical.ZonedTimestampType;
+
+public abstract class FlinkTypeVisitor<T> implements LogicalTypeVisitor<T> {
+
+ // ------------------------- Unsupported types ------------------------------
+
+ @Override
+ public T visit(ZonedTimestampType zonedTimestampType) {
+ throw new UnsupportedOperationException("Unsupported ZonedTimestampType.");
+ }
+
+ @Override
+ public T visit(YearMonthIntervalType yearMonthIntervalType) {
+ throw new UnsupportedOperationException("Unsupported YearMonthIntervalType.");
+ }
+
+ @Override
+ public T visit(DayTimeIntervalType dayTimeIntervalType) {
+ throw new UnsupportedOperationException("Unsupported DayTimeIntervalType.");
+ }
+
+ @Override
+ public T visit(DistinctType distinctType) {
+ throw new UnsupportedOperationException("Unsupported DistinctType.");
+ }
+
+ @Override
+ public T visit(StructuredType structuredType) {
+ throw new UnsupportedOperationException("Unsupported StructuredType.");
+ }
+
+ @Override
+ public T visit(NullType nullType) {
+ throw new UnsupportedOperationException("Unsupported NullType.");
+ }
+
+ @Override
+ public T visit(RawType<?> rawType) {
+ throw new UnsupportedOperationException("Unsupported RawType.");
+ }
+
+ @Override
+ public T visit(SymbolType<?> symbolType) {
+ throw new UnsupportedOperationException("Unsupported SymbolType.");
+ }
+
+ @Override
+ public T visit(LogicalType other) {
+ throw new UnsupportedOperationException("Unsupported type: " + other);
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java
new file mode 100644
index 0000000000..ca5c566bfa
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import java.util.Map;
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.iceberg.DistributionMode;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.flink.sink.shuffle.StatisticsType;
+
+/**
+ * A class for common Iceberg configs for Flink writes.
+ *
+ * <p>If a config is set at multiple levels, the following order of precedence is used (top to
+ * bottom):
+ *
+ * <ol>
+ * <li>Write options
+ * <li>flink ReadableConfig
+ * <li>Table metadata
+ * </ol>
+ *
+ * The most specific value is set in write options and takes precedence over all other configs. If
+ * no write option is provided, this class checks the flink configuration for any overrides. If no
+ * applicable value is found in the write options, this class uses the table metadata.
+ *
+ * <p>Note this class is NOT meant to be serialized.
+ */
+public class FlinkWriteConf {
+
+ private final FlinkConfParser confParser;
+
+ public FlinkWriteConf(
+ Table table, Map<String, String> writeOptions, ReadableConfig readableConfig) {
+ this.confParser = new FlinkConfParser(table, writeOptions, readableConfig);
+ }
+
+ public FlinkWriteConf(Map<String, String> writeOptions, ReadableConfig readableConfig) {
+ this.confParser = new FlinkConfParser(writeOptions, readableConfig);
+ }
+
+ public boolean overwriteMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.OVERWRITE_MODE.key())
+ .flinkConfig(FlinkWriteOptions.OVERWRITE_MODE)
+ .defaultValue(FlinkWriteOptions.OVERWRITE_MODE.defaultValue())
+ .parse();
+ }
+
+ public boolean upsertMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.WRITE_UPSERT_ENABLED.key())
+ .flinkConfig(FlinkWriteOptions.WRITE_UPSERT_ENABLED)
+ .tableProperty(TableProperties.UPSERT_ENABLED)
+ .defaultValue(TableProperties.UPSERT_ENABLED_DEFAULT)
+ .parse();
+ }
+
+ public FileFormat dataFileFormat() {
+ String valueAsString =
+ confParser
+ .stringConf()
+ .option(FlinkWriteOptions.WRITE_FORMAT.key())
+ .flinkConfig(FlinkWriteOptions.WRITE_FORMAT)
+ .tableProperty(TableProperties.DEFAULT_FILE_FORMAT)
+ .defaultValue(TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)
+ .parse();
+ return FileFormat.fromString(valueAsString);
+ }
+
+ public long targetDataFileSize() {
+ return confParser
+ .longConf()
+ .option(FlinkWriteOptions.TARGET_FILE_SIZE_BYTES.key())
+ .flinkConfig(FlinkWriteOptions.TARGET_FILE_SIZE_BYTES)
+ .tableProperty(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES)
+ .defaultValue(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT)
+ .parse();
+ }
+
+ public String parquetCompressionCodec() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
+ .tableProperty(TableProperties.PARQUET_COMPRESSION)
+ .defaultValue(TableProperties.PARQUET_COMPRESSION_DEFAULT)
+ .parse();
+ }
+
+ public String parquetCompressionLevel() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_LEVEL.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_LEVEL)
+ .tableProperty(TableProperties.PARQUET_COMPRESSION_LEVEL)
+ .defaultValue(TableProperties.PARQUET_COMPRESSION_LEVEL_DEFAULT)
+ .parseOptional();
+ }
+
+ public String avroCompressionCodec() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
+ .tableProperty(TableProperties.AVRO_COMPRESSION)
+ .defaultValue(TableProperties.AVRO_COMPRESSION_DEFAULT)
+ .parse();
+ }
+
+ public String avroCompressionLevel() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_LEVEL.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_LEVEL)
+ .tableProperty(TableProperties.AVRO_COMPRESSION_LEVEL)
+ .defaultValue(TableProperties.AVRO_COMPRESSION_LEVEL_DEFAULT)
+ .parseOptional();
+ }
+
+ public String orcCompressionCodec() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
+ .tableProperty(TableProperties.ORC_COMPRESSION)
+ .defaultValue(TableProperties.ORC_COMPRESSION_DEFAULT)
+ .parse();
+ }
+
+ public String orcCompressionStrategy() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.COMPRESSION_STRATEGY.key())
+ .flinkConfig(FlinkWriteOptions.COMPRESSION_STRATEGY)
+ .tableProperty(TableProperties.ORC_COMPRESSION_STRATEGY)
+ .defaultValue(TableProperties.ORC_COMPRESSION_STRATEGY_DEFAULT)
+ .parse();
+ }
+
+ public DistributionMode distributionMode() {
+ String modeName =
+ confParser
+ .stringConf()
+ .option(FlinkWriteOptions.DISTRIBUTION_MODE.key())
+ .flinkConfig(FlinkWriteOptions.DISTRIBUTION_MODE)
+ .tableProperty(TableProperties.WRITE_DISTRIBUTION_MODE)
+ .defaultValue(TableProperties.WRITE_DISTRIBUTION_MODE_NONE)
+ .parse();
+ return DistributionMode.fromName(modeName);
+ }
+
+ public StatisticsType rangeDistributionStatisticsType() {
+ String name =
+ confParser
+ .stringConf()
+ .option(FlinkWriteOptions.RANGE_DISTRIBUTION_STATISTICS_TYPE.key())
+ .flinkConfig(FlinkWriteOptions.RANGE_DISTRIBUTION_STATISTICS_TYPE)
+ .defaultValue(FlinkWriteOptions.RANGE_DISTRIBUTION_STATISTICS_TYPE.defaultValue())
+ .parse();
+ return StatisticsType.valueOf(name);
+ }
+
+ public double rangeDistributionSortKeyBaseWeight() {
+ return confParser
+ .doubleConf()
+ .option(FlinkWriteOptions.RANGE_DISTRIBUTION_SORT_KEY_BASE_WEIGHT.key())
+ .flinkConfig(FlinkWriteOptions.RANGE_DISTRIBUTION_SORT_KEY_BASE_WEIGHT)
+ .defaultValue(FlinkWriteOptions.RANGE_DISTRIBUTION_SORT_KEY_BASE_WEIGHT.defaultValue())
+ .parse();
+ }
+
+ public int workerPoolSize() {
+ return confParser
+ .intConf()
+ .flinkConfig(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE)
+ .defaultValue(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.defaultValue())
+ .parse();
+ }
+
+ public String branch() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.BRANCH.key())
+ .defaultValue(FlinkWriteOptions.BRANCH.defaultValue())
+ .parse();
+ }
+
+ public String uidSuffix() {
+ return confParser
+ .stringConf()
+ .option(FlinkWriteOptions.UID_SUFFIX.key())
+ .defaultValue(FlinkWriteOptions.UID_SUFFIX.defaultValue())
+ .parse();
+ }
+
+ public Integer writeParallelism() {
+ return confParser.intConf().option(FlinkWriteOptions.WRITE_PARALLELISM.key()).parseOptional();
+ }
+
+ public boolean expireSnapshotsMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.EXPIRE_SNAPSHOTS_ENABLE.key())
+ .flinkConfig(FlinkWriteOptions.EXPIRE_SNAPSHOTS_ENABLE)
+ .defaultValue(FlinkWriteOptions.EXPIRE_SNAPSHOTS_ENABLE.defaultValue())
+ .parse();
+ }
+
+ public boolean deleteOrphanFilesMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.DELETE_ORPHAN_FILES_ENABLE.key())
+ .flinkConfig(FlinkWriteOptions.DELETE_ORPHAN_FILES_ENABLE)
+ .defaultValue(FlinkWriteOptions.DELETE_ORPHAN_FILES_ENABLE.defaultValue())
+ .parse();
+ }
+
+ public boolean compactMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.COMPACTION_ENABLE.key())
+ .flinkConfig(FlinkWriteOptions.COMPACTION_ENABLE)
+ .defaultValue(FlinkWriteOptions.COMPACTION_ENABLE.defaultValue())
+ .parse();
+ }
+
+ public boolean convertEqualityDeletesMode() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.CONVERT_EQUALITY_DELETES_ENABLE.key())
+ .flinkConfig(FlinkWriteOptions.CONVERT_EQUALITY_DELETES_ENABLE)
+ .defaultValue(FlinkWriteOptions.CONVERT_EQUALITY_DELETES_ENABLE.defaultValue())
+ .parse();
+ }
+
+ /**
+ * NOTE: This may be removed or changed in a future release. This value specifies the interval for
+ * refreshing the table instances in sink writer subtasks. If not specified then the default
+ * behavior is to not refresh the table.
+ *
+ * @return the interval for refreshing the table in sink writer subtasks
+ */
+ @Experimental
+ public Duration tableRefreshInterval() {
+ return confParser
+ .durationConf()
+ .option(FlinkWriteOptions.TABLE_REFRESH_INTERVAL.key())
+ .flinkConfig(FlinkWriteOptions.TABLE_REFRESH_INTERVAL)
+ .parseOptional();
+ }
+
+ public boolean parquetShredVariants() {
+ return confParser
+ .booleanConf()
+ .option(FlinkWriteOptions.SHRED_VARIANTS.key())
+ .tableProperty(TableProperties.PARQUET_SHRED_VARIANTS)
+ .defaultValue(TableProperties.PARQUET_SHRED_VARIANTS_DEFAULT)
+ .parse();
+ }
+
+ public int parquetVariantInferenceBufferSize() {
+ return confParser
+ .intConf()
+ .option(FlinkWriteOptions.VARIANT_INFERENCE_BUFFER_SIZE.key())
+ .tableProperty(TableProperties.PARQUET_VARIANT_BUFFER_SIZE)
+ .defaultValue(TableProperties.PARQUET_VARIANT_BUFFER_SIZE_DEFAULT)
+ .parse();
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java
new file mode 100644
index 0000000000..6ad4bf30bb
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.time.Duration;
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletesConfig;
+import org.apache.iceberg.flink.maintenance.api.DeleteOrphanFilesConfig;
+import org.apache.iceberg.flink.maintenance.api.ExpireSnapshotsConfig;
+import org.apache.iceberg.flink.maintenance.api.RewriteDataFilesConfig;
+import org.apache.iceberg.flink.sink.shuffle.StatisticsType;
+
+/** Flink sink write options */
+public class FlinkWriteOptions {
+
+ private FlinkWriteOptions() {}
+
+ // File format for write operations(default: Table write.format.default )
+ public static final ConfigOption<String> WRITE_FORMAT =
+ ConfigOptions.key("write-format").stringType().noDefaultValue();
+
+ // Overrides this table's write.target-file-size-bytes
+ public static final ConfigOption<Long> TARGET_FILE_SIZE_BYTES =
+ ConfigOptions.key("target-file-size-bytes").longType().noDefaultValue();
+
+ // Overrides this table's write.<FILE_FORMAT>.compression-codec
+ public static final ConfigOption<String> COMPRESSION_CODEC =
+ ConfigOptions.key("compression-codec").stringType().noDefaultValue();
+
+ // Overrides this table's write.<FILE_FORMAT>.compression-level
+ public static final ConfigOption<String> COMPRESSION_LEVEL =
+ ConfigOptions.key("compression-level").stringType().noDefaultValue();
+
+ // Overrides this table's write.<FILE_FORMAT>.compression-strategy
+ public static final ConfigOption<String> COMPRESSION_STRATEGY =
+ ConfigOptions.key("compression-strategy").stringType().noDefaultValue();
+
+ // Overrides this table's write.upsert.enabled
+ public static final ConfigOption<Boolean> WRITE_UPSERT_ENABLED =
+ ConfigOptions.key("upsert-enabled").booleanType().noDefaultValue();
+
+ public static final ConfigOption<Boolean> OVERWRITE_MODE =
+ ConfigOptions.key("overwrite-enabled").booleanType().defaultValue(false);
+
+ // Overrides the table's write.distribution-mode
+ public static final ConfigOption<String> DISTRIBUTION_MODE =
+ ConfigOptions.key("distribution-mode").stringType().noDefaultValue();
+
+ public static final ConfigOption<String> RANGE_DISTRIBUTION_STATISTICS_TYPE =
+ ConfigOptions.key("range-distribution-statistics-type")
+ .stringType()
+ .defaultValue(StatisticsType.Auto.name())
+ .withDescription("Type of statistics collection: Auto, Map, Sketch");
+
+ public static final ConfigOption<Double> RANGE_DISTRIBUTION_SORT_KEY_BASE_WEIGHT =
+ ConfigOptions.key("range-distribution-sort-key-base-weight")
+ .doubleType()
+ .defaultValue(0.0d)
+ .withDescription(
+ "Base weight for every sort key relative to target weight per writer task");
+
+ // Branch to write to
+ public static final ConfigOption<String> BRANCH =
+ ConfigOptions.key("branch").stringType().defaultValue(SnapshotRef.MAIN_BRANCH);
+
+ public static final ConfigOption<Integer> WRITE_PARALLELISM =
+ ConfigOptions.key("write-parallelism").intType().noDefaultValue();
+
+ public static final ConfigOption<Boolean> COMPACTION_ENABLE =
+ ConfigOptions.key(RewriteDataFilesConfig.PREFIX + "enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDeprecatedKeys("compaction-enabled");
+
+ public static final ConfigOption<Boolean> EXPIRE_SNAPSHOTS_ENABLE =
+ ConfigOptions.key(ExpireSnapshotsConfig.PREFIX + "enabled").booleanType().defaultValue(false);
+
+ public static final ConfigOption<Boolean> DELETE_ORPHAN_FILES_ENABLE =
+ ConfigOptions.key(DeleteOrphanFilesConfig.PREFIX + "enabled")
+ .booleanType()
+ .defaultValue(false);
+
+ public static final ConfigOption<Boolean> CONVERT_EQUALITY_DELETES_ENABLE =
+ ConfigOptions.key(ConvertEqualityDeletesConfig.PREFIX + "enabled")
+ .booleanType()
+ .defaultValue(false);
+
+ @Experimental
+ public static final ConfigOption<Duration> TABLE_REFRESH_INTERVAL =
+ ConfigOptions.key("table-refresh-interval").durationType().noDefaultValue();
+
+ // specify the uidSuffix to be used for the underlying IcebergSink
+ public static final ConfigOption<String> UID_SUFFIX =
+ ConfigOptions.key("uid-suffix").stringType().defaultValue("");
+
+ public static final ConfigOption<Boolean> SHRED_VARIANTS =
+ ConfigOptions.key("shred-variants").booleanType().defaultValue(false);
+
+ public static final ConfigOption<Integer> VARIANT_INFERENCE_BUFFER_SIZE =
+ ConfigOptions.key("variant-inference-buffer-size").intType().noDefaultValue();
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java
new file mode 100644
index 0000000000..a03a66929e
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.catalog.UniqueConstraint;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.abilities.SupportsOverwrite;
+import org.apache.flink.table.connector.sink.abilities.SupportsPartitioning;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.legacy.api.TableSchema;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.iceberg.common.DynConstructors;
+import org.apache.iceberg.flink.sink.FlinkSink;
+import org.apache.iceberg.flink.sink.IcebergSink;
+import org.apache.iceberg.flink.sink.dynamic.DynamicIcebergSink;
+import org.apache.iceberg.flink.sink.dynamic.DynamicRecordGenerator;
+import org.apache.iceberg.flink.sink.dynamic.DynamicTableRecordGenerator;
+import org.apache.iceberg.flink.sink.dynamic.TableCreator;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.util.PropertyUtil;
+
+public class IcebergTableSink implements DynamicTableSink, SupportsPartitioning, SupportsOverwrite {
+ private final TableLoader tableLoader;
+ private final CatalogLoader catalogLoader;
+
+ @SuppressWarnings("deprecation")
+ @Deprecated
+ private final TableSchema tableSchema;
+
+ private final ResolvedSchema resolvedSchema;
+ private final ReadableConfig readableConfig;
+ private final Map<String, String> writeProps;
+ private final String dynamicRecordGeneratorImpl;
+ private boolean overwrite = false;
+ private boolean useDynamicSink = false;
+
+ private IcebergTableSink(IcebergTableSink toCopy) {
+ this.tableLoader = toCopy.tableLoader;
+ this.catalogLoader = toCopy.catalogLoader;
+ this.tableSchema = toCopy.tableSchema;
+ this.resolvedSchema = toCopy.resolvedSchema;
+ this.overwrite = toCopy.overwrite;
+ this.readableConfig = toCopy.readableConfig;
+ this.writeProps = toCopy.writeProps;
+ this.dynamicRecordGeneratorImpl = toCopy.dynamicRecordGeneratorImpl;
+ this.useDynamicSink = toCopy.useDynamicSink;
+ }
+
+ /**
+ * @deprecated since 1.10.0, will be removed in 2.0.0. Use {@link #IcebergTableSink(TableLoader,
+ * ResolvedSchema, ReadableConfig, Map)} instead
+ */
+ @Deprecated
+ public IcebergTableSink(
+ TableLoader tableLoader,
+ TableSchema tableSchema,
+ ReadableConfig readableConfig,
+ Map<String, String> writeProps) {
+ this.tableLoader = tableLoader;
+ this.catalogLoader = null;
+ this.tableSchema = tableSchema;
+ this.resolvedSchema = null;
+ this.readableConfig = readableConfig;
+ this.writeProps = writeProps;
+ this.dynamicRecordGeneratorImpl = null;
+ }
+
+ public IcebergTableSink(
+ TableLoader tableLoader,
+ ResolvedSchema resolvedSchema,
+ ReadableConfig readableConfig,
+ Map<String, String> writeProps) {
+ this.tableLoader = tableLoader;
+ this.catalogLoader = null;
+ this.tableSchema = null;
+ this.resolvedSchema = resolvedSchema;
+ this.readableConfig = readableConfig;
+ this.writeProps = writeProps;
+ this.dynamicRecordGeneratorImpl = null;
+ }
+
+ public IcebergTableSink(
+ CatalogLoader catalogLoader,
+ String dynamicRecordGeneratorImpl,
+ ResolvedSchema resolvedSchema,
+ ReadableConfig readableConfig,
+ Map<String, String> writeProps) {
+ this.tableLoader = null;
+ this.catalogLoader = catalogLoader;
+ this.dynamicRecordGeneratorImpl = dynamicRecordGeneratorImpl;
+ this.readableConfig = readableConfig;
+ this.writeProps = writeProps;
+ this.tableSchema = null;
+ this.resolvedSchema = resolvedSchema;
+ this.useDynamicSink = true;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
+ Preconditions.checkState(
+ !overwrite || context.isBounded(),
+ "Unbounded data stream doesn't support overwrite operation.");
+
+ return (DataStreamSinkProvider)
+ (providerContext, dataStream) -> {
+ if (useDynamicSink) {
+ return createDynamicIcebergSink(dataStream);
+ }
+
+ ResolvedSchema physicalColumnsOnlySchema = null;
+ List<String> equalityColumns;
+ if (resolvedSchema != null) {
+ physicalColumnsOnlySchema =
+ ResolvedSchema.of(
+ resolvedSchema.getColumns().stream()
+ .filter(Column::isPhysical)
+ .collect(Collectors.toList()));
+
+ equalityColumns =
+ physicalColumnsOnlySchema
+ .getPrimaryKey()
+ .map(UniqueConstraint::getColumns)
+ .orElseGet(ImmutableList::of);
+ } else {
+ equalityColumns =
+ tableSchema
+ .getPrimaryKey()
+ .map(org.apache.flink.table.legacy.api.constraints.UniqueConstraint::getColumns)
+ .orElseGet(ImmutableList::of);
+ }
+
+ if (readableConfig.get(FlinkConfigOptions.TABLE_EXEC_ICEBERG_USE_V2_SINK)) {
+ return createIcebergSink(dataStream, equalityColumns, physicalColumnsOnlySchema);
+ } else {
+ return createLegacySink(dataStream, equalityColumns, physicalColumnsOnlySchema);
+ }
+ };
+ }
+
+ @Override
+ public void applyStaticPartition(Map<String, String> partition) {
+ // The flink's PartitionFanoutWriter will handle the static partition write policy
+ // automatically.
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode(ChangelogMode requestedMode) {
+ ChangelogMode.Builder builder = ChangelogMode.newBuilder();
+ for (RowKind kind : requestedMode.getContainedKinds()) {
+ builder.addContainedKind(kind);
+ }
+ return builder.build();
+ }
+
+ @Override
+ public DynamicTableSink copy() {
+ return new IcebergTableSink(this);
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "Iceberg table sink";
+ }
+
+ @Override
+ public void applyOverwrite(boolean newOverwrite) {
+ this.overwrite = newOverwrite;
+ }
+
+ private DataStreamSink<?> createLegacySink(
+ DataStream<RowData> dataStream,
+ List<String> equalityColumns,
+ ResolvedSchema physicalColumnsOnlySchema) {
+ FlinkSink.Builder builder =
+ FlinkSink.forRowData(dataStream)
+ .tableLoader(tableLoader)
+ .equalityFieldColumns(equalityColumns)
+ .overwrite(overwrite)
+ .setAll(writeProps)
+ .flinkConf(readableConfig);
+
+ if (physicalColumnsOnlySchema != null) {
+ builder = builder.resolvedSchema(physicalColumnsOnlySchema);
+ } else {
+ builder = builder.tableSchema(tableSchema);
+ }
+
+ return builder.append();
+ }
+
+ private DataStreamSink<?> createIcebergSink(
+ DataStream<RowData> dataStream,
+ List<String> equalityColumns,
+ ResolvedSchema physicalColumnsOnlySchema) {
+ IcebergSink.Builder builder =
+ IcebergSink.forRowData(dataStream)
+ .tableLoader(tableLoader)
+ .equalityFieldColumns(equalityColumns)
+ .overwrite(overwrite)
+ .setAll(writeProps)
+ .flinkConf(readableConfig);
+
+ if (physicalColumnsOnlySchema != null) {
+ builder = builder.resolvedSchema(physicalColumnsOnlySchema);
+ } else {
+ builder = builder.tableSchema(tableSchema);
+ }
+
+ return builder.append();
+ }
+
+ private DataStreamSink<?> createDynamicIcebergSink(DataStream<RowData> dataStream) {
+ Preconditions.checkArgument(
+ catalogLoader != null && dynamicRecordGeneratorImpl != null,
+ "Invalid value catalogLoader: %s, DynamicRecordGenerator Implementation class: %s. "
+ + "Both should be not null to use dynamic iceberg sink.",
+ catalogLoader,
+ dynamicRecordGeneratorImpl);
+
+ TableCreator tableCreator = createTableCreator();
+ DynamicRecordGenerator<RowData> generator =
+ createDynamicRecordGenerator(dynamicRecordGeneratorImpl);
+
+ DynamicIcebergSink.Builder<RowData> builder =
+ DynamicIcebergSink.forInput(dataStream)
+ .generator(generator)
+ .catalogLoader(catalogLoader)
+ .setAll(writeProps)
+ .tableCreator(tableCreator)
+ .flinkConf(readableConfig);
+
+ return builder.append();
+ }
+
+ private TableCreator createTableCreator() {
+ final Map<String, String> tableProperties =
+ PropertyUtil.propertiesWithPrefix(writeProps, "table.props.");
+ final String location = writeProps.get("location");
+
+ return (catalog, identifier, schema, spec) ->
+ catalog
+ .buildTable(identifier, schema)
+ .withPartitionSpec(spec)
+ .withLocation(location)
+ .withProperties(tableProperties)
+ .create();
+ }
+
+ private DynamicTableRecordGenerator createDynamicRecordGenerator(String generatorImpl) {
+ RowType rowType = (RowType) resolvedSchema.toSourceRowDataType().getLogicalType();
+
+ DynConstructors.Ctor<DynamicTableRecordGenerator> ctor;
+
+ try {
+ ctor =
+ DynConstructors.builder(DynamicTableRecordGenerator.class)
+ .loader(IcebergTableSink.class.getClassLoader())
+ .impl(generatorImpl, RowType.class)
+ .impl(generatorImpl, RowType.class, Map.class, Configuration.class)
+ .buildChecked();
+ return ctor.newInstance(rowType, writeProps, fromReadableConfig());
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(
+ String.format("Class %s does not implement DynamicRecordGeneratorSQL", generatorImpl), e);
+ } catch (Exception e) {
+ throw new RuntimeException(
+ String.format("Failed to instantiate DynamicRecordGeneratorSQL %s", generatorImpl), e);
+ }
+ }
+
+ private Configuration fromReadableConfig() {
+ return readableConfig instanceof Configuration
+ ? (Configuration) readableConfig
+ : Configuration.fromMap(readableConfig.toMap());
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java
new file mode 100644
index 0000000000..f920959632
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.lang.reflect.Array;
+import java.nio.ByteBuffer;
+import java.time.LocalDateTime;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
+import org.apache.iceberg.util.UUIDUtil;
+
+public class RowDataWrapper implements StructLike {
+
+ private final LogicalType[] types;
+ private final PositionalGetter<?>[] getters;
+ private RowData rowData = null;
+
+ public RowDataWrapper(RowType rowType, Types.StructType struct) {
+ int size = rowType.getFieldCount();
+
+ types = (LogicalType[]) Array.newInstance(LogicalType.class, size);
+ getters = (PositionalGetter[]) Array.newInstance(PositionalGetter.class, size);
+
+ for (int i = 0; i < size; i++) {
+ types[i] = rowType.getTypeAt(i);
+ PositionalGetter<?> getter = buildGetter(types[i], struct.fields().get(i).type());
+ if (getter == null) {
+ // Pre-build the Flink field getter once instead of recreating it on every access.
+ RowData.FieldGetter fieldGetter = FlinkRowData.createFieldGetter(types[i], i);
+ getter = (row, pos) -> fieldGetter.getFieldOrNull(row);
+ }
+
+ getters[i] = getter;
+ }
+ }
+
+ public RowDataWrapper wrap(RowData data) {
+ this.rowData = data;
+ return this;
+ }
+
+ @Override
+ public int size() {
+ return types.length;
+ }
+
+ @Override
+ public <T> T get(int pos, Class<T> javaClass) {
+ if (rowData.isNullAt(pos)) {
+ return null;
+ }
+
+ return javaClass.cast(getters[pos].get(rowData, pos));
+ }
+
+ @Override
+ public <T> void set(int pos, T value) {
+ throw new UnsupportedOperationException(
+ "Could not set a field in the RowDataWrapper because rowData is read-only");
+ }
+
+ private interface PositionalGetter<T> {
+ T get(RowData data, int pos);
+ }
+
+ private static PositionalGetter<?> buildGetter(LogicalType logicalType, Type type) {
+ switch (logicalType.getTypeRoot()) {
+ case TINYINT:
+ return (row, pos) -> (int) row.getByte(pos);
+ case SMALLINT:
+ return (row, pos) -> (int) row.getShort(pos);
+ case CHAR:
+ case VARCHAR:
+ return (row, pos) -> row.getString(pos).toString();
+
+ case BINARY:
+ case VARBINARY:
+ if (Type.TypeID.UUID == type.typeId()) {
+ return (row, pos) -> UUIDUtil.convert(row.getBinary(pos));
+ } else {
+ return (row, pos) -> ByteBuffer.wrap(row.getBinary(pos));
+ }
+
+ case DECIMAL:
+ DecimalType decimalType = (DecimalType) logicalType;
+ return (row, pos) ->
+ row.getDecimal(pos, decimalType.getPrecision(), decimalType.getScale()).toBigDecimal();
+
+ case TIME_WITHOUT_TIME_ZONE:
+ // Time in RowData is in milliseconds (Integer), while iceberg's time is microseconds
+ // (Long).
+ return (row, pos) -> ((long) row.getInt(pos)) * 1_000;
+
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ TimestampType timestampType = (TimestampType) logicalType;
+ if (type.typeId() == Type.TypeID.TIMESTAMP_NANO) {
+ return (row, pos) -> {
+ LocalDateTime localDateTime =
+ row.getTimestamp(pos, timestampType.getPrecision()).toLocalDateTime();
+ return DateTimeUtil.nanosFromTimestamp(localDateTime);
+ };
+ } else {
+ return (row, pos) -> {
+ LocalDateTime localDateTime =
+ row.getTimestamp(pos, timestampType.getPrecision()).toLocalDateTime();
+ return DateTimeUtil.microsFromTimestamp(localDateTime);
+ };
+ }
+
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ LocalZonedTimestampType lzTs = (LocalZonedTimestampType) logicalType;
+ if (type.typeId() == Type.TypeID.TIMESTAMP_NANO) {
+ return (row, pos) -> {
+ TimestampData timestampData = row.getTimestamp(pos, lzTs.getPrecision());
+ return timestampData.getMillisecond() * 1_000_000L
+ + timestampData.getNanoOfMillisecond();
+ };
+ } else {
+ return (row, pos) -> {
+ TimestampData timestampData = row.getTimestamp(pos, lzTs.getPrecision());
+ return timestampData.getMillisecond() * 1000L
+ + timestampData.getNanoOfMillisecond() / 1000;
+ };
+ }
+
+ case ROW:
+ RowType rowType = (RowType) logicalType;
+ Types.StructType structType = (Types.StructType) type;
+
+ RowDataWrapper nestedWrapper = new RowDataWrapper(rowType, structType);
+ return (row, pos) -> nestedWrapper.wrap(row.getRow(pos, rowType.getFieldCount()));
+
+ default:
+ return null;
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java
new file mode 100644
index 0000000000..da509451fe
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.hadoop.SerializableConfiguration;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+
+/**
+ * Serializable loader to load an Iceberg {@link Table}. Flink needs to get {@link Table} objects in
+ * the cluster (for example, to get splits), not just on the client side. So we need an Iceberg
+ * table loader to get the {@link Table} object.
+ */
+public interface TableLoader extends Closeable, Serializable, Cloneable {
+
+ void open();
+
+ boolean isOpen();
+
+ Table loadTable();
+
+ /** Clone a TableLoader */
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ TableLoader clone();
+
+ static TableLoader fromCatalog(CatalogLoader catalogLoader, TableIdentifier identifier) {
+ return new CatalogTableLoader(catalogLoader, identifier);
+ }
+
+ static TableLoader fromHadoopTable(String location) {
+ return fromHadoopTable(location, FlinkCatalogFactory.clusterHadoopConf());
+ }
+
+ static TableLoader fromHadoopTable(String location, Configuration hadoopConf) {
+ return new HadoopTableLoader(location, hadoopConf);
+ }
+
+ class HadoopTableLoader implements TableLoader {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String location;
+ private final SerializableConfiguration hadoopConf;
+
+ private transient HadoopTables tables;
+
+ private HadoopTableLoader(String location, Configuration conf) {
+ this.location = location;
+ this.hadoopConf = new SerializableConfiguration(conf);
+ }
+
+ @Override
+ public void open() {
+ tables = new HadoopTables(hadoopConf.get());
+ }
+
+ @Override
+ public boolean isOpen() {
+ return tables != null;
+ }
+
+ @Override
+ public Table loadTable() {
+ FlinkEnvironmentContext.init();
+ return tables.load(location);
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public TableLoader clone() {
+ return new HadoopTableLoader(location, new Configuration(hadoopConf.get()));
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("location", location).toString();
+ }
+ }
+
+ class CatalogTableLoader implements TableLoader {
+
+ private static final long serialVersionUID = 1L;
+
+ private final CatalogLoader catalogLoader;
+ private final String identifier;
+
+ private transient Catalog catalog;
+
+ private CatalogTableLoader(CatalogLoader catalogLoader, TableIdentifier tableIdentifier) {
+ this.catalogLoader = catalogLoader;
+ this.identifier = tableIdentifier.toString();
+ }
+
+ @Override
+ public void open() {
+ catalog = catalogLoader.loadCatalog();
+ }
+
+ @Override
+ public boolean isOpen() {
+ return catalog != null;
+ }
+
+ @Override
+ public Table loadTable() {
+ FlinkEnvironmentContext.init();
+ return catalog.loadTable(TableIdentifier.parse(identifier));
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (catalog instanceof Closeable) {
+ ((Closeable) catalog).close();
+ }
+
+ catalog = null;
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public TableLoader clone() {
+ return new CatalogTableLoader(catalogLoader.clone(), TableIdentifier.parse(identifier));
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("tableIdentifier", identifier)
+ .add("catalogLoader", catalogLoader)
+ .toString();
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java
new file mode 100644
index 0000000000..e5b1186354
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import java.util.List;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.BigIntType;
+import org.apache.flink.table.types.logical.BinaryType;
+import org.apache.flink.table.types.logical.BooleanType;
+import org.apache.flink.table.types.logical.DateType;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.DoubleType;
+import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.NullType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.TimeType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.table.types.logical.VarBinaryType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.flink.table.types.logical.VariantType;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+class TypeToFlinkType extends TypeUtil.SchemaVisitor<LogicalType> {
+ TypeToFlinkType() {}
+
+ @Override
+ public LogicalType schema(Schema schema, LogicalType structType) {
+ return structType;
+ }
+
+ @Override
+ public LogicalType struct(Types.StructType struct, List<LogicalType> fieldResults) {
+ List<Types.NestedField> fields = struct.fields();
+
+ List<RowType.RowField> flinkFields = Lists.newArrayListWithExpectedSize(fieldResults.size());
+ for (int i = 0; i < fields.size(); i += 1) {
+ Types.NestedField field = fields.get(i);
+ LogicalType type = fieldResults.get(i);
+ RowType.RowField flinkField =
+ new RowType.RowField(field.name(), type.copy(field.isOptional()), field.doc());
+ flinkFields.add(flinkField);
+ }
+
+ return new RowType(flinkFields);
+ }
+
+ @Override
+ public LogicalType field(Types.NestedField field, LogicalType fieldResult) {
+ return fieldResult;
+ }
+
+ @Override
+ public LogicalType list(Types.ListType list, LogicalType elementResult) {
+ return new ArrayType(elementResult.copy(list.isElementOptional()));
+ }
+
+ @Override
+ public LogicalType map(Types.MapType map, LogicalType keyResult, LogicalType valueResult) {
+ // keys in map are not allowed to be null.
+ return new MapType(keyResult.copy(false), valueResult.copy(map.isValueOptional()));
+ }
+
+ @Override
+ public LogicalType variant(Types.VariantType variant) {
+ return new VariantType();
+ }
+
+ @Override
+ public LogicalType primitive(Type.PrimitiveType primitive) {
+ switch (primitive.typeId()) {
+ case UNKNOWN:
+ return new NullType();
+ case BOOLEAN:
+ return new BooleanType();
+ case INTEGER:
+ return new IntType();
+ case LONG:
+ return new BigIntType();
+ case FLOAT:
+ return new FloatType();
+ case DOUBLE:
+ return new DoubleType();
+ case DATE:
+ return new DateType();
+ case TIME:
+ // For the type: Flink only support TimeType with default precision (second) now. The
+ // precision of time is
+ // not supported in Flink, so we can think of it as a simple time type directly.
+ // For the data: Flink uses int that support mills to represent time data, so it supports
+ // mills precision.
+ return new TimeType();
+ case TIMESTAMP:
+ Types.TimestampType timestamp = (Types.TimestampType) primitive;
+ if (timestamp.shouldAdjustToUTC()) {
+ // MICROS
+ return new LocalZonedTimestampType(6);
+ } else {
+ // MICROS
+ return new TimestampType(6);
+ }
+ case TIMESTAMP_NANO:
+ Types.TimestampNanoType timestamp9 = (Types.TimestampNanoType) primitive;
+ if (timestamp9.shouldAdjustToUTC()) {
+ // NANOS
+ return new LocalZonedTimestampType(9);
+ } else {
+ // NANOS
+ return new TimestampType(9);
+ }
+ case STRING:
+ return new VarCharType(VarCharType.MAX_LENGTH);
+ case UUID:
+ // UUID length is 16
+ return new BinaryType(16);
+ case FIXED:
+ Types.FixedType fixedType = (Types.FixedType) primitive;
+ return new BinaryType(fixedType.length());
+ case BINARY:
+ return new VarBinaryType(VarBinaryType.MAX_LENGTH);
+ case DECIMAL:
+ Types.DecimalType decimal = (Types.DecimalType) primitive;
+ return new DecimalType(decimal.precision(), decimal.scale());
+ default:
+ throw new UnsupportedOperationException(
+ "Cannot convert unknown type to Flink: " + primitive);
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java
new file mode 100644
index 0000000000..b96b47c5a7
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.actions;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.iceberg.Table;
+
+public class Actions {
+
+ public static final Configuration CONFIG =
+ new Configuration()
+ // disable classloader check as Avro may cache class/object in the serializers.
+ .set(CoreOptions.CHECK_LEAKED_CLASSLOADER, false);
+
+ private final StreamExecutionEnvironment env;
+ private final Table table;
+
+ private Actions(StreamExecutionEnvironment env, Table table) {
+ this.env = env;
+ this.table = table;
+ }
+
+ public static Actions forTable(StreamExecutionEnvironment env, Table table) {
+ return new Actions(env, table);
+ }
+
+ public static Actions forTable(Table table) {
+ return new Actions(StreamExecutionEnvironment.getExecutionEnvironment(CONFIG), table);
+ }
+
+ public RewriteDataFilesAction rewriteDataFiles() {
+ return new RewriteDataFilesAction(env, table);
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java
new file mode 100644
index 0000000000..dcd4ac4e58
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.actions;
+
+import java.util.List;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.iceberg.CombinedScanTask;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.actions.BaseRewriteDataFilesAction;
+import org.apache.iceberg.flink.source.RowDataRewriter;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+public class RewriteDataFilesAction extends BaseRewriteDataFilesAction<RewriteDataFilesAction> {
+
+ private final StreamExecutionEnvironment env;
+ private int maxParallelism;
+
+ public RewriteDataFilesAction(StreamExecutionEnvironment env, Table table) {
+ super(table);
+ this.env = env;
+ this.maxParallelism = env.getParallelism();
+ Preconditions.checkArgument(
+ !TableUtil.supportsRowLineage(table),
+ "Flink does not support compaction on row lineage enabled tables (V3+)");
+ }
+
+ @Override
+ protected FileIO fileIO() {
+ return table().io();
+ }
+
+ @Override
+ protected List<DataFile> rewriteDataForTasks(List<CombinedScanTask> combinedScanTasks) {
+ int size = combinedScanTasks.size();
+ int parallelism = Math.min(size, maxParallelism);
+ DataStream<CombinedScanTask> dataStream = env.fromData(combinedScanTasks);
+ RowDataRewriter rowDataRewriter =
+ new RowDataRewriter(table(), caseSensitive(), fileIO(), encryptionManager());
+ try {
+ return rowDataRewriter.rewriteDataForTasks(dataStream, parallelism);
+ } catch (Exception e) {
+ throw new RuntimeException("Rewrite data file error.", e);
+ }
+ }
+
+ @Override
+ protected RewriteDataFilesAction self() {
+ return this;
+ }
+
+ public RewriteDataFilesAction maxParallelism(int parallelism) {
+ Preconditions.checkArgument(parallelism > 0, "Invalid max parallelism %s", parallelism);
+ this.maxParallelism = parallelism;
+ return this;
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java
new file mode 100644
index 0000000000..1826a267c6
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.NullType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VariantType;
+import org.apache.iceberg.avro.AvroWithPartnerByStructureVisitor;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.util.Pair;
+
+public abstract class AvroWithFlinkSchemaVisitor<T>
+ extends AvroWithPartnerByStructureVisitor<LogicalType, T> {
+
+ @Override
+ protected boolean isVariantType(LogicalType logicalType) {
+ return logicalType instanceof VariantType;
+ }
+
+ @Override
+ protected boolean isStringType(LogicalType logicalType) {
+ return logicalType.getTypeRoot().getFamilies().contains(LogicalTypeFamily.CHARACTER_STRING);
+ }
+
+ @Override
+ protected boolean isMapType(LogicalType logicalType) {
+ return logicalType instanceof MapType;
+ }
+
+ @Override
+ protected LogicalType arrayElementType(LogicalType arrayType) {
+ Preconditions.checkArgument(
+ arrayType instanceof ArrayType, "Invalid array: %s is not an array", arrayType);
+ return ((ArrayType) arrayType).getElementType();
+ }
+
+ @Override
+ protected LogicalType mapKeyType(LogicalType mapType) {
+ Preconditions.checkArgument(isMapType(mapType), "Invalid map: %s is not a map", mapType);
+ return ((MapType) mapType).getKeyType();
+ }
+
+ @Override
+ protected LogicalType mapValueType(LogicalType mapType) {
+ Preconditions.checkArgument(isMapType(mapType), "Invalid map: %s is not a map", mapType);
+ return ((MapType) mapType).getValueType();
+ }
+
+ @Override
+ protected Pair<String, LogicalType> fieldNameAndType(LogicalType structType, int pos) {
+ Preconditions.checkArgument(
+ structType instanceof RowType, "Invalid struct: %s is not a struct", structType);
+ RowType.RowField field = ((RowType) structType).getFields().get(pos);
+ return Pair.of(field.getName(), field.getType());
+ }
+
+ @Override
+ protected LogicalType nullType() {
+ return new NullType();
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java
new file mode 100644
index 0000000000..b1577d3861
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.apache.avro.io.Encoder;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.FieldMetrics;
+import org.apache.iceberg.avro.MetricsAwareDatumWriter;
+import org.apache.iceberg.avro.ValueWriter;
+import org.apache.iceberg.avro.ValueWriters;
+import org.apache.iceberg.flink.FlinkSchemaUtil;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+public class FlinkAvroWriter implements MetricsAwareDatumWriter<RowData> {
+ private final RowType rowType;
+ private ValueWriter<RowData> writer = null;
+
+ public FlinkAvroWriter(RowType rowType) {
+ this.rowType = rowType;
+ }
+
+ public FlinkAvroWriter(org.apache.iceberg.Schema icebergSchema, RowType engineSchema) {
+ this(engineSchema != null ? engineSchema : FlinkSchemaUtil.convert(icebergSchema));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void setSchema(Schema schema) {
+ this.writer =
+ (ValueWriter<RowData>)
+ AvroWithFlinkSchemaVisitor.visit(rowType, schema, new WriteBuilder());
+ }
+
+ @Override
+ public void write(RowData datum, Encoder out) throws IOException {
+ writer.write(datum, out);
+ }
+
+ @Override
+ public Stream<FieldMetrics> metrics() {
+ return writer.metrics();
+ }
+
+ private static class WriteBuilder extends AvroWithFlinkSchemaVisitor<ValueWriter<?>> {
+ @Override
+ public ValueWriter<?> record(
+ LogicalType struct, Schema record, List<String> names, List<ValueWriter<?>> fields) {
+ return FlinkValueWriters.row(
+ fields,
+ IntStream.range(0, names.size())
+ .mapToObj(i -> fieldNameAndType(struct, i).second())
+ .collect(Collectors.toList()));
+ }
+
+ @Override
+ public ValueWriter<?> union(LogicalType type, Schema union, List<ValueWriter<?>> options) {
+ Preconditions.checkArgument(
+ options.contains(ValueWriters.nulls()),
+ "Cannot create writer for non-option union: %s",
+ union);
+ Preconditions.checkArgument(
+ options.size() == 2, "Cannot create writer for non-option union: %s", union);
+ if (union.getTypes().get(0).getType() == Schema.Type.NULL) {
+ return ValueWriters.option(0, options.get(1));
+ } else {
+ return ValueWriters.option(1, options.get(0));
+ }
+ }
+
+ @Override
+ public ValueWriter<?> array(LogicalType sArray, Schema array, ValueWriter<?> elementWriter) {
+ return FlinkValueWriters.array(elementWriter, arrayElementType(sArray));
+ }
+
+ @Override
+ public ValueWriter<?> map(LogicalType sMap, Schema map, ValueWriter<?> valueReader) {
+ return FlinkValueWriters.map(
+ FlinkValueWriters.strings(), mapKeyType(sMap), valueReader, mapValueType(sMap));
+ }
+
+ @Override
+ public ValueWriter<?> map(
+ LogicalType sMap, Schema map, ValueWriter<?> keyWriter, ValueWriter<?> valueWriter) {
+ return FlinkValueWriters.arrayMap(
+ keyWriter, mapKeyType(sMap), valueWriter, mapValueType(sMap));
+ }
+
+ @Override
+ public ValueWriter<?> variant(
+ LogicalType variant, ValueWriter<?> metadataWriter, ValueWriter<?> valueWriter) {
+ return FlinkValueWriters.variants();
+ }
+
+ @Override
+ public ValueWriter<?> primitive(LogicalType type, Schema primitive) {
+ org.apache.avro.LogicalType logicalType = primitive.getLogicalType();
+ if (logicalType != null) {
+ switch (logicalType.getName()) {
+ case "date":
+ return ValueWriters.ints();
+
+ case "time-micros":
+ return FlinkValueWriters.timeMicros();
+
+ case "timestamp-micros":
+ return FlinkValueWriters.timestampMicros();
+
+ case "timestamp-nanos":
+ return FlinkValueWriters.timestampNanos();
+
+ case "decimal":
+ LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
+ return FlinkValueWriters.decimal(decimal.getPrecision(), decimal.getScale());
+
+ case "uuid":
+ return FlinkValueWriters.uuids();
+
+ default:
+ throw new IllegalArgumentException("Unsupported logical type: " + logicalType);
+ }
+ }
+
+ switch (primitive.getType()) {
+ case NULL:
+ return ValueWriters.nulls();
+ case BOOLEAN:
+ return ValueWriters.booleans();
+ case INT:
+ switch (type.getTypeRoot()) {
+ case TINYINT:
+ return ValueWriters.tinyints();
+ case SMALLINT:
+ return ValueWriters.shorts();
+ default:
+ return ValueWriters.ints();
+ }
+ case LONG:
+ return ValueWriters.longs();
+ case FLOAT:
+ return ValueWriters.floats();
+ case DOUBLE:
+ return ValueWriters.doubles();
+ case STRING:
+ return FlinkValueWriters.strings();
+ case FIXED:
+ return ValueWriters.fixed(primitive.getFixedSize());
+ case BYTES:
+ return ValueWriters.bytes();
+ default:
+ throw new IllegalArgumentException("Unsupported type: " + primitive);
+ }
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkFormatModels.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkFormatModels.java
new file mode 100644
index 0000000000..747a461868
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkFormatModels.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.function.Function;
+import java.util.function.UnaryOperator;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.avro.AvroFormatModel;
+import org.apache.iceberg.formats.FormatModelRegistry;
+import org.apache.iceberg.orc.ORCFormatModel;
+import org.apache.iceberg.parquet.ParquetFormatModel;
+
+public class FlinkFormatModels {
+ public static void register() {
+ FormatModelRegistry.register(
+ ParquetFormatModel.create(
+ RowData.class,
+ RowType.class,
+ FlinkParquetWriters::buildWriter,
+ (icebergSchema, fileSchema, engineSchema, idToConstant) ->
+ FlinkParquetReaders.buildReader(icebergSchema, fileSchema, idToConstant),
+ new FlinkVariantShreddingAnalyzer(),
+ (Function<RowType, UnaryOperator<RowData>>)
+ rowType -> new RowDataSerializer(rowType)::copy));
+
+ FormatModelRegistry.register(
+ AvroFormatModel.create(
+ RowData.class,
+ RowType.class,
+ (icebergSchema, fileSchema, engineSchema) ->
+ new FlinkAvroWriter(icebergSchema, engineSchema),
+ (icebergSchema, fileSchema, engineSchema, idToConstant) ->
+ FlinkPlannedAvroReader.create(icebergSchema, idToConstant)));
+
+ FormatModelRegistry.register(
+ ORCFormatModel.create(
+ RowData.class,
+ RowType.class,
+ (icebergSchema, fileSchema, engineSchema) ->
+ FlinkOrcWriter.buildWriter(engineSchema, icebergSchema),
+ (icebergSchema, fileSchema, engineSchema, idToConstant) ->
+ new FlinkOrcReader(icebergSchema, fileSchema, idToConstant)));
+ }
+
+ private FlinkFormatModels() {}
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java
new file mode 100644
index 0000000000..77f16bfdb2
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.orc.OrcRowReader;
+import org.apache.iceberg.orc.OrcSchemaWithTypeVisitor;
+import org.apache.iceberg.orc.OrcValueReader;
+import org.apache.iceberg.orc.OrcValueReaders;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.storage.ql.exec.vector.StructColumnVector;
+import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
+
+public class FlinkOrcReader implements OrcRowReader<RowData> {
+ private final OrcValueReader<?> reader;
+
+ public FlinkOrcReader(Schema iSchema, TypeDescription readSchema) {
+ this(iSchema, readSchema, ImmutableMap.of());
+ }
+
+ public FlinkOrcReader(Schema iSchema, TypeDescription readSchema, Map<Integer, ?> idToConstant) {
+ this.reader =
+ OrcSchemaWithTypeVisitor.visit(iSchema, readSchema, new ReadBuilder(idToConstant));
+ }
+
+ @Override
+ public RowData read(VectorizedRowBatch batch, int row) {
+ return (RowData) reader.read(new StructColumnVector(batch.size, batch.cols), row);
+ }
+
+ @Override
+ public void setBatchContext(long batchOffsetInFile) {
+ reader.setBatchContext(batchOffsetInFile);
+ }
+
+ private static class ReadBuilder extends OrcSchemaWithTypeVisitor<OrcValueReader<?>> {
+ private final Map<Integer, ?> idToConstant;
+
+ private ReadBuilder(Map<Integer, ?> idToConstant) {
+ this.idToConstant = idToConstant;
+ }
+
+ @Override
+ public OrcValueReader<RowData> record(
+ Types.StructType iStruct,
+ TypeDescription record,
+ List<String> names,
+ List<OrcValueReader<?>> fields) {
+ return FlinkOrcReaders.struct(record, fields, iStruct, idToConstant);
+ }
+
+ @Override
+ public OrcValueReader<ArrayData> list(
+ Types.ListType iList, TypeDescription array, OrcValueReader<?> elementReader) {
+ return FlinkOrcReaders.array(elementReader);
+ }
+
+ @Override
+ public OrcValueReader<MapData> map(
+ Types.MapType iMap,
+ TypeDescription map,
+ OrcValueReader<?> keyReader,
+ OrcValueReader<?> valueReader) {
+ return FlinkOrcReaders.map(keyReader, valueReader);
+ }
+
+ @Override
+ public OrcValueReader<?> primitive(Type.PrimitiveType iPrimitive, TypeDescription primitive) {
+ switch (iPrimitive.typeId()) {
+ case BOOLEAN:
+ return OrcValueReaders.booleans();
+ case INTEGER:
+ return OrcValueReaders.ints();
+ case LONG:
+ return OrcValueReaders.longs();
+ case FLOAT:
+ return OrcValueReaders.floats();
+ case DOUBLE:
+ return OrcValueReaders.doubles();
+ case DATE:
+ return FlinkOrcReaders.dates();
+ case TIME:
+ return FlinkOrcReaders.times();
+ case TIMESTAMP:
+ Types.TimestampType timestampType = (Types.TimestampType) iPrimitive;
+ if (timestampType.shouldAdjustToUTC()) {
+ return FlinkOrcReaders.timestampTzs();
+ } else {
+ return FlinkOrcReaders.timestamps();
+ }
+ case TIMESTAMP_NANO:
+ Types.TimestampNanoType timestampNanoType = (Types.TimestampNanoType) iPrimitive;
+ if (timestampNanoType.shouldAdjustToUTC()) {
+ return FlinkOrcReaders.timestampTzs();
+ } else {
+ return FlinkOrcReaders.timestamps();
+ }
+ case STRING:
+ return FlinkOrcReaders.strings();
+ case UUID:
+ case FIXED:
+ case BINARY:
+ return OrcValueReaders.bytes();
+ case DECIMAL:
+ Types.DecimalType decimalType = (Types.DecimalType) iPrimitive;
+ return FlinkOrcReaders.decimals(decimalType.precision(), decimalType.scale());
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid iceberg type %s corresponding to ORC type %s", iPrimitive, primitive));
+ }
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java
new file mode 100644
index 0000000000..c5c958fbdb
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericArrayData;
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.iceberg.orc.OrcValueReader;
+import org.apache.iceberg.orc.OrcValueReaders;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.storage.ql.exec.vector.BytesColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ColumnVector;
+import org.apache.orc.storage.ql.exec.vector.DecimalColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ListColumnVector;
+import org.apache.orc.storage.ql.exec.vector.LongColumnVector;
+import org.apache.orc.storage.ql.exec.vector.MapColumnVector;
+import org.apache.orc.storage.ql.exec.vector.TimestampColumnVector;
+import org.apache.orc.storage.serde2.io.HiveDecimalWritable;
+
+class FlinkOrcReaders {
+ private FlinkOrcReaders() {}
+
+ static OrcValueReader<StringData> strings() {
+ return StringReader.INSTANCE;
+ }
+
+ static OrcValueReader<Integer> dates() {
+ return DateReader.INSTANCE;
+ }
+
+ static OrcValueReader<DecimalData> decimals(int precision, int scale) {
+ if (precision <= 18) {
+ return new Decimal18Reader(precision, scale);
+ } else if (precision <= 38) {
+ return new Decimal38Reader(precision, scale);
+ } else {
+ throw new IllegalArgumentException("Invalid precision: " + precision);
+ }
+ }
+
+ static OrcValueReader<Integer> times() {
+ return TimeReader.INSTANCE;
+ }
+
+ static OrcValueReader<TimestampData> timestamps() {
+ return TimestampReader.INSTANCE;
+ }
+
+ static OrcValueReader<TimestampData> timestampTzs() {
+ return TimestampTzReader.INSTANCE;
+ }
+
+ static <T> OrcValueReader<ArrayData> array(OrcValueReader<T> elementReader) {
+ return new ArrayReader<>(elementReader);
+ }
+
+ public static <K, V> OrcValueReader<MapData> map(
+ OrcValueReader<K> keyReader, OrcValueReader<V> valueReader) {
+ return new MapReader<>(keyReader, valueReader);
+ }
+
+ public static OrcValueReader<RowData> struct(
+ TypeDescription record,
+ List<OrcValueReader<?>> readers,
+ Types.StructType struct,
+ Map<Integer, ?> idToConstant) {
+ return new StructReader(record, readers, struct, idToConstant);
+ }
+
+ private static class StringReader implements OrcValueReader<StringData> {
+ private static final StringReader INSTANCE = new StringReader();
+
+ @Override
+ public StringData nonNullRead(ColumnVector vector, int row) {
+ BytesColumnVector bytesVector = (BytesColumnVector) vector;
+ return StringData.fromBytes(
+ bytesVector.vector[row], bytesVector.start[row], bytesVector.length[row]);
+ }
+ }
+
+ private static class DateReader implements OrcValueReader<Integer> {
+ private static final DateReader INSTANCE = new DateReader();
+
+ @Override
+ public Integer nonNullRead(ColumnVector vector, int row) {
+ return (int) ((LongColumnVector) vector).vector[row];
+ }
+ }
+
+ private static class Decimal18Reader implements OrcValueReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal18Reader(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData nonNullRead(ColumnVector vector, int row) {
+ HiveDecimalWritable value = ((DecimalColumnVector) vector).vector[row];
+
+ // The hive ORC writer may will adjust the scale of decimal data.
+ Preconditions.checkArgument(
+ value.precision() <= precision,
+ "Cannot read value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ value);
+
+ return DecimalData.fromUnscaledLong(value.serialize64(scale), precision, scale);
+ }
+ }
+
+ private static class Decimal38Reader implements OrcValueReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal38Reader(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData nonNullRead(ColumnVector vector, int row) {
+ BigDecimal value =
+ ((DecimalColumnVector) vector).vector[row].getHiveDecimal().bigDecimalValue();
+
+ Preconditions.checkArgument(
+ value.precision() <= precision,
+ "Cannot read value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ value);
+
+ return DecimalData.fromBigDecimal(value, precision, scale);
+ }
+ }
+
+ private static class TimeReader implements OrcValueReader<Integer> {
+ private static final TimeReader INSTANCE = new TimeReader();
+
+ @Override
+ public Integer nonNullRead(ColumnVector vector, int row) {
+ long micros = ((LongColumnVector) vector).vector[row];
+ // Flink only support time mills, just erase micros.
+ return (int) (micros / 1000);
+ }
+ }
+
+ private static class TimestampReader implements OrcValueReader<TimestampData> {
+ private static final TimestampReader INSTANCE = new TimestampReader();
+
+ @Override
+ public TimestampData nonNullRead(ColumnVector vector, int row) {
+ TimestampColumnVector tcv = (TimestampColumnVector) vector;
+ LocalDateTime localDate =
+ Instant.ofEpochSecond(Math.floorDiv(tcv.time[row], 1_000), tcv.nanos[row])
+ .atOffset(ZoneOffset.UTC)
+ .toLocalDateTime();
+ return TimestampData.fromLocalDateTime(localDate);
+ }
+ }
+
+ private static class TimestampTzReader implements OrcValueReader<TimestampData> {
+ private static final TimestampTzReader INSTANCE = new TimestampTzReader();
+
+ @Override
+ public TimestampData nonNullRead(ColumnVector vector, int row) {
+ TimestampColumnVector tcv = (TimestampColumnVector) vector;
+ Instant instant =
+ Instant.ofEpochSecond(Math.floorDiv(tcv.time[row], 1_000), tcv.nanos[row])
+ .atOffset(ZoneOffset.UTC)
+ .toInstant();
+ return TimestampData.fromInstant(instant);
+ }
+ }
+
+ private static class ArrayReader<T> implements OrcValueReader<ArrayData> {
+ private final OrcValueReader<T> elementReader;
+
+ private ArrayReader(OrcValueReader<T> elementReader) {
+ this.elementReader = elementReader;
+ }
+
+ @Override
+ public ArrayData nonNullRead(ColumnVector vector, int row) {
+ ListColumnVector listVector = (ListColumnVector) vector;
+ int offset = (int) listVector.offsets[row];
+ int length = (int) listVector.lengths[row];
+ List<T> elements = Lists.newArrayListWithExpectedSize(length);
+ for (int c = 0; c < length; ++c) {
+ elements.add(elementReader.read(listVector.child, offset + c));
+ }
+ return new GenericArrayData(elements.toArray());
+ }
+
+ @Override
+ public void setBatchContext(long batchOffsetInFile) {
+ elementReader.setBatchContext(batchOffsetInFile);
+ }
+ }
+
+ private static class MapReader<K, V> implements OrcValueReader<MapData> {
+ private final OrcValueReader<K> keyReader;
+ private final OrcValueReader<V> valueReader;
+
+ private MapReader(OrcValueReader<K> keyReader, OrcValueReader<V> valueReader) {
+ this.keyReader = keyReader;
+ this.valueReader = valueReader;
+ }
+
+ @Override
+ public MapData nonNullRead(ColumnVector vector, int row) {
+ MapColumnVector mapVector = (MapColumnVector) vector;
+ int offset = (int) mapVector.offsets[row];
+ long length = mapVector.lengths[row];
+
+ Map<K, V> map = Maps.newHashMap();
+ for (int c = 0; c < length; c++) {
+ K key = keyReader.read(mapVector.keys, offset + c);
+ V value = valueReader.read(mapVector.values, offset + c);
+ map.put(key, value);
+ }
+
+ return new GenericMapData(map);
+ }
+
+ @Override
+ public void setBatchContext(long batchOffsetInFile) {
+ keyReader.setBatchContext(batchOffsetInFile);
+ valueReader.setBatchContext(batchOffsetInFile);
+ }
+ }
+
+ private static class StructReader extends OrcValueReaders.StructReader<RowData> {
+ private final int numFields;
+
+ StructReader(
+ TypeDescription record,
+ List<OrcValueReader<?>> readers,
+ Types.StructType struct,
+ Map<Integer, ?> idToConstant) {
+ super(record, readers, struct, idToConstant);
+ this.numFields = struct.fields().size();
+ }
+
+ @Override
+ protected RowData create() {
+ return new GenericRowData(numFields);
+ }
+
+ @Override
+ protected void set(RowData struct, int pos, Object value) {
+ ((GenericRowData) struct).setField(pos, value);
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java
new file mode 100644
index 0000000000..c1b46252e1
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.util.Deque;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.iceberg.FieldMetrics;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.data.orc.GenericOrcWriters;
+import org.apache.iceberg.flink.FlinkSchemaUtil;
+import org.apache.iceberg.orc.OrcRowWriter;
+import org.apache.iceberg.orc.OrcValueWriter;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
+
+public class FlinkOrcWriter implements OrcRowWriter<RowData> {
+ private final FlinkOrcWriters.RowDataWriter writer;
+
+ private FlinkOrcWriter(RowType rowType, Schema iSchema) {
+ this.writer =
+ (FlinkOrcWriters.RowDataWriter)
+ FlinkSchemaVisitor.visit(rowType, iSchema, new WriteBuilder());
+ }
+
+ public static OrcRowWriter<RowData> buildWriter(RowType rowType, Schema iSchema) {
+ return new FlinkOrcWriter(
+ rowType != null ? rowType : FlinkSchemaUtil.convert(iSchema), iSchema);
+ }
+
+ @Override
+ public void write(RowData row, VectorizedRowBatch output) {
+ Preconditions.checkArgument(row != null, "value must not be null");
+ writer.writeRow(row, output);
+ }
+
+ @Override
+ public List<OrcValueWriter<?>> writers() {
+ return writer.writers();
+ }
+
+ @Override
+ public Stream<FieldMetrics<?>> metrics() {
+ return writer.metrics();
+ }
+
+ private static class WriteBuilder extends FlinkSchemaVisitor<OrcValueWriter<?>> {
+ private final Deque<Integer> fieldIds = Lists.newLinkedList();
+
+ private WriteBuilder() {}
+
+ @Override
+ public void beforeField(Types.NestedField field) {
+ fieldIds.push(field.fieldId());
+ }
+
+ @Override
+ public void afterField(Types.NestedField field) {
+ fieldIds.pop();
+ }
+
+ @Override
+ public OrcValueWriter<RowData> record(
+ Types.StructType iStruct, List<OrcValueWriter<?>> results, List<LogicalType> fieldType) {
+ return FlinkOrcWriters.struct(results, fieldType);
+ }
+
+ @Override
+ public OrcValueWriter<?> map(
+ Types.MapType iMap,
+ OrcValueWriter<?> key,
+ OrcValueWriter<?> value,
+ LogicalType keyType,
+ LogicalType valueType) {
+ return FlinkOrcWriters.map(key, value, keyType, valueType);
+ }
+
+ @Override
+ public OrcValueWriter<?> list(
+ Types.ListType iList, OrcValueWriter<?> element, LogicalType elementType) {
+ return FlinkOrcWriters.list(element, elementType);
+ }
+
+ @Override
+ public OrcValueWriter<?> primitive(Type.PrimitiveType iPrimitive, LogicalType flinkPrimitive) {
+ switch (iPrimitive.typeId()) {
+ case BOOLEAN:
+ return GenericOrcWriters.booleans();
+ case INTEGER:
+ switch (flinkPrimitive.getTypeRoot()) {
+ case TINYINT:
+ return GenericOrcWriters.bytes();
+ case SMALLINT:
+ return GenericOrcWriters.shorts();
+ }
+ return GenericOrcWriters.ints();
+ case LONG:
+ return GenericOrcWriters.longs();
+ case FLOAT:
+ Preconditions.checkArgument(
+ fieldIds.peek() != null,
+ String.format(
+ "[BUG] Cannot find field id for primitive field with type %s. This is likely because id "
+ + "information is not properly pushed during schema visiting.",
+ iPrimitive));
+ return GenericOrcWriters.floats(fieldIds.peek());
+ case DOUBLE:
+ Preconditions.checkArgument(
+ fieldIds.peek() != null,
+ String.format(
+ "[BUG] Cannot find field id for primitive field with type %s. This is likely because id "
+ + "information is not properly pushed during schema visiting.",
+ iPrimitive));
+ return GenericOrcWriters.doubles(fieldIds.peek());
+ case DATE:
+ return FlinkOrcWriters.dates();
+ case TIME:
+ return FlinkOrcWriters.times();
+ case TIMESTAMP:
+ Types.TimestampType timestampType = (Types.TimestampType) iPrimitive;
+ if (timestampType.shouldAdjustToUTC()) {
+ return FlinkOrcWriters.timestampTzs();
+ } else {
+ return FlinkOrcWriters.timestamps();
+ }
+ case TIMESTAMP_NANO:
+ Types.TimestampNanoType timestampNanoType = (Types.TimestampNanoType) iPrimitive;
+ if (timestampNanoType.shouldAdjustToUTC()) {
+ return FlinkOrcWriters.timestampNanoTzs();
+ } else {
+ return FlinkOrcWriters.timestampNanos();
+ }
+ case STRING:
+ return FlinkOrcWriters.strings();
+ case UUID:
+ case FIXED:
+ case BINARY:
+ return GenericOrcWriters.byteArrays();
+ case DECIMAL:
+ Types.DecimalType decimalType = (Types.DecimalType) iPrimitive;
+ return FlinkOrcWriters.decimals(decimalType.precision(), decimalType.scale());
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid iceberg type %s corresponding to Flink logical type %s",
+ iPrimitive, flinkPrimitive));
+ }
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java
new file mode 100644
index 0000000000..bf19a46c05
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.iceberg.FieldMetrics;
+import org.apache.iceberg.data.orc.GenericOrcWriters;
+import org.apache.iceberg.flink.FlinkRowData;
+import org.apache.iceberg.orc.OrcValueWriter;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.orc.storage.common.type.HiveDecimal;
+import org.apache.orc.storage.ql.exec.vector.BytesColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ColumnVector;
+import org.apache.orc.storage.ql.exec.vector.DecimalColumnVector;
+import org.apache.orc.storage.ql.exec.vector.ListColumnVector;
+import org.apache.orc.storage.ql.exec.vector.LongColumnVector;
+import org.apache.orc.storage.ql.exec.vector.MapColumnVector;
+import org.apache.orc.storage.ql.exec.vector.TimestampColumnVector;
+
+class FlinkOrcWriters {
+
+ private FlinkOrcWriters() {}
+
+ static OrcValueWriter<StringData> strings() {
+ return StringWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<Integer> dates() {
+ return DateWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<Integer> times() {
+ return TimeWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<TimestampData> timestamps() {
+ return TimestampWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<TimestampData> timestampTzs() {
+ return TimestampTzWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<TimestampData> timestampNanos() {
+ return TimestampNanoWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<TimestampData> timestampNanoTzs() {
+ return TimestampNanoTzWriter.INSTANCE;
+ }
+
+ static OrcValueWriter<DecimalData> decimals(int precision, int scale) {
+ if (precision <= 18) {
+ return new Decimal18Writer(precision, scale);
+ } else if (precision <= 38) {
+ return new Decimal38Writer(precision, scale);
+ } else {
+ throw new IllegalArgumentException("Invalid precision: " + precision);
+ }
+ }
+
+ static <T> OrcValueWriter<ArrayData> list(
+ OrcValueWriter<T> elementWriter, LogicalType elementType) {
+ return new ListWriter<>(elementWriter, elementType);
+ }
+
+ static <K, V> OrcValueWriter<MapData> map(
+ OrcValueWriter<K> keyWriter,
+ OrcValueWriter<V> valueWriter,
+ LogicalType keyType,
+ LogicalType valueType) {
+ return new MapWriter<>(keyWriter, valueWriter, keyType, valueType);
+ }
+
+ static OrcValueWriter<RowData> struct(List<OrcValueWriter<?>> writers, List<LogicalType> types) {
+ int[] fieldIndexes = new int[writers.size()];
+ int fieldIndex = 0;
+ List<LogicalType> logicalTypes = Lists.newArrayList();
+ for (int i = 0; i < types.size(); i += 1) {
+ LogicalType logicalType = types.get(i);
+ if (!logicalType.is(LogicalTypeRoot.NULL)) {
+ fieldIndexes[fieldIndex] = i;
+ fieldIndex += 1;
+ logicalTypes.add(logicalType);
+ }
+ }
+
+ return new RowDataWriter(fieldIndexes, writers, logicalTypes);
+ }
+
+ private static class StringWriter implements OrcValueWriter<StringData> {
+ private static final StringWriter INSTANCE = new StringWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, StringData data, ColumnVector output) {
+ byte[] value = data.toBytes();
+ ((BytesColumnVector) output).setRef(rowId, value, 0, value.length);
+ }
+ }
+
+ private static class DateWriter implements OrcValueWriter<Integer> {
+ private static final DateWriter INSTANCE = new DateWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, Integer data, ColumnVector output) {
+ ((LongColumnVector) output).vector[rowId] = data;
+ }
+ }
+
+ private static class TimeWriter implements OrcValueWriter<Integer> {
+ private static final TimeWriter INSTANCE = new TimeWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, Integer millis, ColumnVector output) {
+ // The time in flink is in millisecond, while the standard time in iceberg is microsecond.
+ // So we need to transform it to microsecond.
+ ((LongColumnVector) output).vector[rowId] = millis * 1000L;
+ }
+ }
+
+ private static class TimestampWriter implements OrcValueWriter<TimestampData> {
+ private static final TimestampWriter INSTANCE = new TimestampWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
+ TimestampColumnVector cv = (TimestampColumnVector) output;
+ cv.setIsUTC(true);
+ // millis
+ OffsetDateTime offsetDateTime = data.toInstant().atOffset(ZoneOffset.UTC);
+ cv.time[rowId] =
+ offsetDateTime.toEpochSecond() * 1_000 + offsetDateTime.getNano() / 1_000_000;
+ // truncate nanos to only keep microsecond precision.
+ cv.nanos[rowId] = (offsetDateTime.getNano() / 1_000) * 1_000;
+ }
+ }
+
+ private static class TimestampTzWriter implements OrcValueWriter<TimestampData> {
+ private static final TimestampTzWriter INSTANCE = new TimestampTzWriter();
+
+ @SuppressWarnings("JavaInstantGetSecondsGetNano")
+ @Override
+ public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
+ TimestampColumnVector cv = (TimestampColumnVector) output;
+ // millis
+ Instant instant = data.toInstant();
+ cv.time[rowId] = instant.toEpochMilli();
+ // truncate nanos to only keep microsecond precision.
+ cv.nanos[rowId] = (instant.getNano() / 1_000) * 1_000;
+ }
+ }
+
+ private static class TimestampNanoWriter implements OrcValueWriter<TimestampData> {
+ private static final TimestampNanoWriter INSTANCE = new TimestampNanoWriter();
+
+ @Override
+ public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
+ TimestampColumnVector cv = (TimestampColumnVector) output;
+ cv.setIsUTC(true);
+ // millis
+ OffsetDateTime offsetDateTime = data.toInstant().atOffset(ZoneOffset.UTC);
+ cv.time[rowId] =
+ offsetDateTime.toEpochSecond() * 1_000 + offsetDateTime.getNano() / 1_000_000;
+ cv.nanos[rowId] = offsetDateTime.getNano();
+ }
+ }
+
+ private static class TimestampNanoTzWriter implements OrcValueWriter<TimestampData> {
+ private static final TimestampNanoTzWriter INSTANCE = new TimestampNanoTzWriter();
+
+ @SuppressWarnings("JavaInstantGetSecondsGetNano")
+ @Override
+ public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
+ TimestampColumnVector cv = (TimestampColumnVector) output;
+ // millis
+ Instant instant = data.toInstant();
+ cv.time[rowId] = instant.toEpochMilli();
+ cv.nanos[rowId] = instant.getNano();
+ }
+ }
+
+ private static class Decimal18Writer implements OrcValueWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal18Writer(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public void nonNullWrite(int rowId, DecimalData data, ColumnVector output) {
+ Preconditions.checkArgument(
+ scale == data.scale(),
+ "Cannot write value as decimal(%s,%s), wrong scale: %s",
+ precision,
+ scale,
+ data);
+ Preconditions.checkArgument(
+ data.precision() <= precision,
+ "Cannot write value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ data);
+
+ ((DecimalColumnVector) output)
+ .vector[rowId].setFromLongAndScale(data.toUnscaledLong(), data.scale());
+ }
+ }
+
+ private static class Decimal38Writer implements OrcValueWriter<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ Decimal38Writer(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public void nonNullWrite(int rowId, DecimalData data, ColumnVector output) {
+ Preconditions.checkArgument(
+ scale == data.scale(),
+ "Cannot write value as decimal(%s,%s), wrong scale: %s",
+ precision,
+ scale,
+ data);
+ Preconditions.checkArgument(
+ data.precision() <= precision,
+ "Cannot write value as decimal(%s,%s), too large: %s",
+ precision,
+ scale,
+ data);
+
+ ((DecimalColumnVector) output)
+ .vector[rowId].set(HiveDecimal.create(data.toBigDecimal(), false));
+ }
+ }
+
+ static class ListWriter<T> implements OrcValueWriter<ArrayData> {
+ private final OrcValueWriter<T> elementWriter;
+ private final ArrayData.ElementGetter elementGetter;
+
+ ListWriter(OrcValueWriter<T> elementWriter, LogicalType elementType) {
+ this.elementWriter = elementWriter;
+ this.elementGetter = ArrayData.createElementGetter(elementType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void nonNullWrite(int rowId, ArrayData data, ColumnVector output) {
+ ListColumnVector cv = (ListColumnVector) output;
+ cv.lengths[rowId] = data.size();
+ cv.offsets[rowId] = cv.childCount;
+ cv.childCount = (int) (cv.childCount + cv.lengths[rowId]);
+ // make sure the child is big enough.
+ growColumnVector(cv.child, cv.childCount);
+
+ for (int e = 0; e < cv.lengths[rowId]; ++e) {
+ Object value = elementGetter.getElementOrNull(data, e);
+ elementWriter.write((int) (e + cv.offsets[rowId]), (T) value, cv.child);
+ }
+ }
+
+ @Override
+ public Stream<FieldMetrics<?>> metrics() {
+ return elementWriter.metrics();
+ }
+ }
+
+ static class MapWriter<K, V> implements OrcValueWriter<MapData> {
+ private final OrcValueWriter<K> keyWriter;
+ private final OrcValueWriter<V> valueWriter;
+ private final ArrayData.ElementGetter keyGetter;
+ private final ArrayData.ElementGetter valueGetter;
+
+ MapWriter(
+ OrcValueWriter<K> keyWriter,
+ OrcValueWriter<V> valueWriter,
+ LogicalType keyType,
+ LogicalType valueType) {
+ this.keyWriter = keyWriter;
+ this.valueWriter = valueWriter;
+ this.keyGetter = ArrayData.createElementGetter(keyType);
+ this.valueGetter = ArrayData.createElementGetter(valueType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void nonNullWrite(int rowId, MapData data, ColumnVector output) {
+ MapColumnVector cv = (MapColumnVector) output;
+ ArrayData keyArray = data.keyArray();
+ ArrayData valArray = data.valueArray();
+
+ // record the length and start of the list elements
+ cv.lengths[rowId] = data.size();
+ cv.offsets[rowId] = cv.childCount;
+ cv.childCount = (int) (cv.childCount + cv.lengths[rowId]);
+ // make sure the child is big enough
+ growColumnVector(cv.keys, cv.childCount);
+ growColumnVector(cv.values, cv.childCount);
+ // Add each element
+ for (int e = 0; e < cv.lengths[rowId]; ++e) {
+ int pos = (int) (e + cv.offsets[rowId]);
+ keyWriter.write(pos, (K) keyGetter.getElementOrNull(keyArray, e), cv.keys);
+ valueWriter.write(pos, (V) valueGetter.getElementOrNull(valArray, e), cv.values);
+ }
+ }
+
+ @Override
+ public Stream<FieldMetrics<?>> metrics() {
+ return Stream.concat(keyWriter.metrics(), valueWriter.metrics());
+ }
+ }
+
+ static class RowDataWriter extends GenericOrcWriters.StructWriter<RowData> {
+ private final List<RowData.FieldGetter> fieldGetters;
+
+ RowDataWriter(int[] fieldIndexes, List<OrcValueWriter<?>> writers, List<LogicalType> types) {
+ super(writers);
+
+ this.fieldGetters = Lists.newArrayListWithExpectedSize(types.size());
+ for (int i = 0; i < types.size(); i++) {
+ fieldGetters.add(FlinkRowData.createFieldGetter(types.get(i), fieldIndexes[i]));
+ }
+ }
+
+ @Override
+ protected Object get(RowData struct, int index) {
+ return fieldGetters.get(index).getFieldOrNull(struct);
+ }
+ }
+
+ private static void growColumnVector(ColumnVector cv, int requestedSize) {
+ if (cv.isNull.length < requestedSize) {
+ // Use growth factor of 3 to avoid frequent array allocations
+ cv.ensureSize(requestedSize * 3, true);
+ }
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java
new file mode 100644
index 0000000000..49d96d1325
--- /dev/null
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java
@@ -0,0 +1,901 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.data;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericArrayData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RawValueData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.types.variant.BinaryVariant;
+import org.apache.flink.types.variant.Variant;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.parquet.ParquetSchemaUtil;
+import org.apache.iceberg.parquet.ParquetValueReader;
+import org.apache.iceberg.parquet.ParquetValueReaders;
+import org.apache.iceberg.parquet.ParquetVariantReaders.DelegatingValueReader;
+import org.apache.iceberg.parquet.ParquetVariantVisitor;
+import org.apache.iceberg.parquet.TypeWithSchemaVisitor;
+import org.apache.iceberg.parquet.VariantReaderBuilder;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ArrayUtil;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+public class FlinkParquetReaders {
+ private FlinkParquetReaders() {}
+
+ public static ParquetValueReader<RowData> buildReader(
+ Schema expectedSchema, MessageType fileSchema) {
+ return buildReader(expectedSchema, fileSchema, ImmutableMap.of());
+ }
+
+ @SuppressWarnings("unchecked")
+ public static ParquetValueReader<RowData> buildReader(
+ Schema expectedSchema, MessageType fileSchema, Map<Integer, ?> idToConstant) {
+ return (ParquetValueReader<RowData>)
+ TypeWithSchemaVisitor.visit(
+ expectedSchema.asStruct(), fileSchema, new ReadBuilder(fileSchema, idToConstant));
+ }
+
+ private static class ReadBuilder extends TypeWithSchemaVisitor<ParquetValueReader<?>> {
+ private final MessageType type;
+ private final Map<Integer, ?> idToConstant;
+
+ ReadBuilder(MessageType type, Map<Integer, ?> idToConstant) {
+ this.type = type;
+ this.idToConstant = idToConstant;
+ }
+
+ @Override
+ public ParquetValueReader<RowData> message(
+ Types.StructType expected, MessageType message, List<ParquetValueReader<?>> fieldReaders) {
+ return struct(expected, message.asGroupType(), fieldReaders);
+ }
+
+ @Override
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ public ParquetValueReader<RowData> struct(
+ Types.StructType expected, GroupType struct, List<ParquetValueReader<?>> fieldReaders) {
+
+ if (null == expected) {
+ return new RowDataReader(ImmutableList.of());
+ }
+
+ // match the expected struct's order
+ Map<Integer, ParquetValueReader<?>> readersById = Maps.newHashMap();
+ List<Type> fields = struct.getFields();
+ for (int i = 0; i < fields.size(); i += 1) {
+ Type fieldType = fields.get(i);
+ if (fieldReaders.get(i) != null) {
+ int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1;
+ if (fieldType.getId() != null) {
+ int id = fieldType.getId().intValue();
+ readersById.put(id, ParquetValueReaders.option(fieldType, fieldD, fieldReaders.get(i)));
+ }
+ }
+ }
+
+ int constantDefinitionLevel = type.getMaxDefinitionLevel(currentPath());
+ List<Types.NestedField> expectedFields = expected.fields();
+ List<ParquetValueReader<?>> reorderedFields =
+ Lists.newArrayListWithExpectedSize(expectedFields.size());
+ for (Types.NestedField field : expectedFields) {
+ int id = field.fieldId();
+ ParquetValueReader<?> reader =
+ ParquetValueReaders.replaceWithMetadataReader(
+ id, readersById.get(id), idToConstant, constantDefinitionLevel);
+ reorderedFields.add(defaultReader(field, reader, constantDefinitionLevel));
+ }
+
+ return new RowDataReader(reorderedFields);
+ }
+
+ private ParquetValueReader<?> defaultReader(
+ Types.NestedField field, ParquetValueReader<?> reader, int constantDL) {
+ if (reader != null) {
+ return reader;
+ } else if (field.initialDefault() != null) {
+ return ParquetValueReaders.constant(
+ RowDataUtil.convertConstant(field.type(), field.initialDefault()), constantDL);
+ } else if (field.isOptional()) {
+ return ParquetValueReaders.nulls();
+ }
+
+ throw new IllegalArgumentException(String.format("Missing required field: %s", field.name()));
+ }
+
+ @Override
+ public ParquetVariantVisitor<ParquetValueReader<?>> variantVisitor() {
+ return new VariantReaderBuilder(type, Arrays.asList(currentPath()));
+ }
+
+ @Override
+ public ParquetValueReader<?> variant(
+ Types.VariantType expected, GroupType variant, ParquetValueReader<?> variantReader) {
+ return new VariantReader(variantReader);
+ }
+
+ @Override
+ public ParquetValueReader<?> list(
+ Types.ListType expectedList, GroupType array, ParquetValueReader<?> elementReader) {
+ if (expectedList == null) {
+ return null;
+ }
+
+ String[] repeatedPath = currentPath();
+
+ int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
+ int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;
+
+ Type elementType = ParquetSchemaUtil.determineListElementType(array);
+ int elementD = type.getMaxDefinitionLevel(path(elementType.getName())) - 1;
+
+ return new ArrayReader<>(
+ repeatedD, repeatedR, ParquetValueReaders.option(elementType, elementD, elementReader));
+ }
+
+ @Override
+ public ParquetValueReader<?> map(
+ Types.MapType expectedMap,
+ GroupType map,
+ ParquetValueReader<?> keyReader,
+ ParquetValueReader<?> valueReader) {
+ if (expectedMap == null) {
+ return null;
+ }
+
+ GroupType repeatedKeyValue = map.getFields().get(0).asGroupType();
+ String[] repeatedPath = currentPath();
+
+ int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
+ int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;
+
+ Type keyType = repeatedKeyValue.getType(0);
+ int keyD = type.getMaxDefinitionLevel(path(keyType.getName())) - 1;
+ Type valueType = repeatedKeyValue.getType(1);
+ int valueD = type.getMaxDefinitionLevel(path(valueType.getName())) - 1;
+
+ return new MapReader<>(
+ repeatedD,
+ repeatedR,
+ ParquetValueReaders.option(keyType, keyD, keyReader),
+ ParquetValueReaders.option(valueType, valueD, valueReader));
+ }
+
+ private static class LogicalTypeAnnotationParquetValueReaderVisitor
+ implements LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<ParquetValueReader<?>> {
+
+ private final PrimitiveType primitive;
+ private final ColumnDescriptor desc;
+ private final org.apache.iceberg.types.Type.PrimitiveType expected;
+
+ LogicalTypeAnnotationParquetValueReaderVisitor(
+ PrimitiveType primitive,
+ ColumnDescriptor desc,
+ org.apache.iceberg.types.Type.PrimitiveType expected) {
+ this.primitive = primitive;
+ this.desc = desc;
+ this.expected = expected;
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.StringLogicalTypeAnnotation stringLogicalType) {
+ return Optional.of(new StringReader(desc));
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) {
+ return Optional.of(new StringReader(desc));
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.JsonLogicalTypeAnnotation jsonLogicalType) {
+ return Optional.of(new StringReader(desc));
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ DecimalLogicalTypeAnnotation decimalLogicalType) {
+ switch (primitive.getPrimitiveTypeName()) {
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ return Optional.of(
+ new BinaryDecimalReader(
+ desc, decimalLogicalType.getPrecision(), decimalLogicalType.getScale()));
+ case INT64:
+ return Optional.of(
+ new LongDecimalReader(
+ desc, decimalLogicalType.getPrecision(), decimalLogicalType.getScale()));
+ case INT32:
+ return Optional.of(
+ new IntegerDecimalReader(
+ desc, decimalLogicalType.getPrecision(), decimalLogicalType.getScale()));
+ }
+
+ return LogicalTypeAnnotation.LogicalTypeAnnotationVisitor.super.visit(decimalLogicalType);
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) {
+ return Optional.of(new ParquetValueReaders.UnboxedReader<>(desc));
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
+ if (timeLogicalType.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS) {
+ return Optional.of(new MillisTimeReader(desc));
+ } else if (timeLogicalType.getUnit() == LogicalTypeAnnotation.TimeUnit.MICROS) {
+ return Optional.of(new LossyMicrosToMillisTimeReader(desc));
+ }
+
+ return LogicalTypeAnnotation.LogicalTypeAnnotationVisitor.super.visit(timeLogicalType);
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) {
+ if (timestampLogicalType.getUnit() == LogicalTypeAnnotation.TimeUnit.MILLIS) {
+ return Optional.of(new MillisToTimestampReader(desc));
+ } else if (timestampLogicalType.getUnit() == LogicalTypeAnnotation.TimeUnit.MICROS) {
+ return Optional.of(new MicrosToTimestampReader(desc));
+ } else if (timestampLogicalType.getUnit() == LogicalTypeAnnotation.TimeUnit.NANOS) {
+ return Optional.of(new NanosToTimestampReader(desc));
+ }
+
+ return LogicalTypeAnnotation.LogicalTypeAnnotationVisitor.super.visit(timestampLogicalType);
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.IntLogicalTypeAnnotation intLogicalType) {
+ int width = intLogicalType.getBitWidth();
+ if (width <= 32) {
+ if (expected.typeId() == Types.LongType.get().typeId()) {
+ return Optional.of(new ParquetValueReaders.IntAsLongReader(desc));
+ } else {
+ return Optional.of(new ParquetValueReaders.UnboxedReader<>(desc));
+ }
+ } else if (width <= 64) {
+ return Optional.of(new ParquetValueReaders.UnboxedReader<>(desc));
+ }
+
+ return LogicalTypeAnnotation.LogicalTypeAnnotationVisitor.super.visit(intLogicalType);
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.BsonLogicalTypeAnnotation bsonLogicalType) {
+ return Optional.of(new ParquetValueReaders.ByteArrayReader(desc));
+ }
+
+ @Override
+ public Optional<ParquetValueReader<?>> visit(
+ LogicalTypeAnnotation.UUIDLogicalTypeAnnotation uuidLogicalType) {
+ return Optional.of(new ParquetValueReaders.ByteArrayReader(desc));
+ }
+ }
+
+ @Override
+ @SuppressWarnings("CyclomaticComplexity")
+ public ParquetValueReader<?> primitive(
+ org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) {
+ if (expected == null) {
+ return null;
+ }
+
+ ColumnDescriptor desc = type.getColumnDescription(currentPath());
+ LogicalTypeAnnotation logicalTypeAnnotation = primitive.getLogicalTypeAnnotation();
+ if (logicalTypeAnnotation != null) {
+ return logicalTypeAnnotation
+ .accept(new LogicalTypeAnnotationParquetValueReaderVisitor(primitive, desc, expected))
+ .orElseThrow(
+ () ->
+ new UnsupportedOperationException(
+ "Unsupported logical type: " + primitive.getLogicalTypeAnnotation()));
+ }
+
+ switch (primitive.getPrimitiveTypeName()) {
+ case FIXED_LEN_BYTE_ARRAY:
+ case BINARY:
+ return new ParquetValueReaders.ByteArrayReader(desc);
+ case INT32:
+ if (expected.typeId() == org.apache.iceberg.types.Type.TypeID.LONG) {
+ return new ParquetValueReaders.IntAsLongReader(desc);
+ } else {
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ }
+ case FLOAT:
+ if (expected.typeId() == org.apache.iceberg.types.Type.TypeID.DOUBLE) {
+ return new ParquetValueReaders.FloatAsDoubleReader(desc);
+ } else {
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ }
+ case BOOLEAN:
+ case INT64:
+ case DOUBLE:
+ return new ParquetValueReaders.UnboxedReader<>(desc);
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + primitive);
+ }
+ }
+ }
+
+ private static class BinaryDecimalReader
+ extends ParquetValueReaders.PrimitiveReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ BinaryDecimalReader(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData read(DecimalData ignored) {
+ Binary binary = column.nextBinary();
+ BigDecimal bigDecimal = new BigDecimal(new BigInteger(binary.getBytes()), scale);
+ return DecimalData.fromBigDecimal(bigDecimal, precision, scale);
+ }
+ }
+
+ private static class IntegerDecimalReader
+ extends ParquetValueReaders.PrimitiveReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ IntegerDecimalReader(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData read(DecimalData ignored) {
+ return DecimalData.fromUnscaledLong(column.nextInteger(), precision, scale);
+ }
+ }
+
+ private static class LongDecimalReader extends ParquetValueReaders.PrimitiveReader<DecimalData> {
+ private final int precision;
+ private final int scale;
+
+ LongDecimalReader(ColumnDescriptor desc, int precision, int scale) {
+ super(desc);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public DecimalData read(DecimalData ignored) {
+ return DecimalData.fromUnscaledLong(column.nextLong(), precision, scale);
+ }
+ }
+
+ private static class NanosToTimestampReader
+ extends ParquetValueReaders.UnboxedReader<TimestampData> {
+ NanosToTimestampReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public TimestampData read(TimestampData ignored) {
+ long value = readLong();
+ return TimestampData.fromEpochMillis(
+ Math.floorDiv(value, 1_000_000L), Math.floorMod(value, 1_000_000));
+ }
+ }
+
+ private static class MicrosToTimestampReader
+ extends ParquetValueReaders.UnboxedReader<TimestampData> {
+ MicrosToTimestampReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public TimestampData read(TimestampData ignored) {
+ long micros = readLong();
+ return TimestampData.fromEpochMillis(
+ Math.floorDiv(micros, 1000L), Math.floorMod(micros, 1000) * 1000);
+ }
+ }
+
+ private static class MillisToTimestampReader
+ extends ParquetValueReaders.UnboxedReader<TimestampData> {
+ MillisToTimestampReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public TimestampData read(TimestampData ignored) {
+ long millis = readLong();
+ return TimestampData.fromEpochMillis(millis);
+ }
+ }
+
+ private static class StringReader extends ParquetValueReaders.PrimitiveReader<StringData> {
+ StringReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public StringData read(StringData ignored) {
+ Binary binary = column.nextBinary();
+ ByteBuffer buffer = binary.toByteBuffer();
+ if (buffer.hasArray()) {
+ return StringData.fromBytes(
+ buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
+ } else {
+ return StringData.fromBytes(binary.getBytes());
+ }
+ }
+ }
+
+ private static class LossyMicrosToMillisTimeReader
+ extends ParquetValueReaders.PrimitiveReader<Integer> {
+ LossyMicrosToMillisTimeReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public Integer read(Integer reuse) {
+ // Discard microseconds since Flink uses millisecond unit for TIME type.
+ return (int) Math.floorDiv(column.nextLong(), 1000L);
+ }
+ }
+
+ private static class MillisTimeReader extends ParquetValueReaders.PrimitiveReader<Integer> {
+ MillisTimeReader(ColumnDescriptor desc) {
+ super(desc);
+ }
+
+ @Override
+ public Integer read(Integer reuse) {
+ return (int) column.nextLong();
+ }
+ }
+
+ private static class ArrayReader<E>
+ extends ParquetValueReaders.RepeatedReader<ArrayData, ReusableArrayData, E> {
+ private int readPos = 0;
+ private int writePos = 0;
+
+ ArrayReader(int definitionLevel, int repetitionLevel, ParquetValueReader<E> reader) {
+ super(definitionLevel, repetitionLevel, reader);
+ }
+
+ @Override
+ protected ReusableArrayData newListData(ArrayData reuse) {
+ this.readPos = 0;
+ this.writePos = 0;
+
+ if (reuse instanceof ReusableArrayData) {
+ return (ReusableArrayData) reuse;
+ } else {
+ return new ReusableArrayData();
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected E getElement(ReusableArrayData list) {
+ E value = null;
+ if (readPos < list.capacity()) {
+ value = (E) list.values[readPos];
+ }
+
+ readPos += 1;
+
+ return value;
+ }
+
+ @Override
+ protected void addElement(ReusableArrayData reused, E element) {
+ if (writePos >= reused.capacity()) {
+ reused.grow();
+ }
+
+ reused.values[writePos] = element;
+
+ writePos += 1;
+ }
+
+ @Override
+ protected ArrayData buildList(ReusableArrayData list) {
+ // Since ReusableArrayData is not accepted by Flink, use GenericArrayData temporarily to walk
+ // around it.
+ // Revert this to use ReusableArrayData once it is fixed in Flink.
+ // For your reference, https://issues.apache.org/jira/browse/FLINK-25238.
+ return new GenericArrayData(Arrays.copyOf(list.values, writePos));
+ }
+ }
+
+ private static class MapReader<K, V>
+ extends ParquetValueReaders.RepeatedKeyValueReader<MapData, ReusableMapData, K, V> {
+ private int readPos = 0;
+ private int writePos = 0;
+
+ private final ParquetValueReaders.ReusableEntry<K, V> entry =
+ new ParquetValueReaders.ReusableEntry<>();
+ private final ParquetValueReaders.ReusableEntry<K, V> nullEntry =
+ new ParquetValueReaders.ReusableEntry<>();
+
+ MapReader(
+ int definitionLevel,
+ int repetitionLevel,
+ ParquetValueReader<K> keyReader,
+ ParquetValueReader<V> valueReader) {
+ super(definitionLevel, repetitionLevel, keyReader, valueReader);
+ }
+
+ @Override
+ protected ReusableMapData newMapData(MapData reuse) {
+ this.readPos = 0;
+ this.writePos = 0;
... 109279 lines suppressed ...