You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2024/04/16 16:10:33 UTC
(iceberg) branch main updated: Flink: Removes Flink version 1.16 (#10154)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new dd194b4391 Flink: Removes Flink version 1.16 (#10154)
dd194b4391 is described below
commit dd194b4391b947ee7caabbefd5e992fd92b9c524
Author: Rodrigo <rm...@apple.com>
AuthorDate: Tue Apr 16 09:10:27 2024 -0700
Flink: Removes Flink version 1.16 (#10154)
---
flink/v1.16/build.gradle | 266 -----
flink/v1.16/flink-runtime/LICENSE | 502 ---------
flink/v1.16/flink-runtime/NOTICE | 91 --
.../iceberg/flink/IcebergConnectorSmokeTest.java | 29 -
.../sink/shuffle/MapRangePartitionerBenchmark.java | 199 ----
.../org/apache/iceberg/flink/CatalogLoader.java | 215 ----
.../org/apache/iceberg/flink/FlinkCatalog.java | 806 --------------
.../apache/iceberg/flink/FlinkCatalogFactory.java | 213 ----
.../org/apache/iceberg/flink/FlinkConfParser.java | 261 -----
.../apache/iceberg/flink/FlinkConfigOptions.java | 107 --
.../iceberg/flink/FlinkDynamicTableFactory.java | 209 ----
.../iceberg/flink/FlinkEnvironmentContext.java | 31 -
.../org/apache/iceberg/flink/FlinkFilters.java | 266 -----
.../org/apache/iceberg/flink/FlinkFixupTypes.java | 50 -
.../org/apache/iceberg/flink/FlinkReadConf.java | 213 ----
.../org/apache/iceberg/flink/FlinkReadOptions.java | 123 --
.../org/apache/iceberg/flink/FlinkSchemaUtil.java | 181 ---
.../apache/iceberg/flink/FlinkSourceFilter.java | 49 -
.../org/apache/iceberg/flink/FlinkTypeToType.java | 199 ----
.../org/apache/iceberg/flink/FlinkTypeVisitor.java | 80 --
.../org/apache/iceberg/flink/FlinkWriteConf.java | 205 ----
.../apache/iceberg/flink/FlinkWriteOptions.java | 73 --
.../org/apache/iceberg/flink/IcebergTableSink.java | 121 --
.../org/apache/iceberg/flink/RowDataWrapper.java | 142 ---
.../java/org/apache/iceberg/flink/TableLoader.java | 159 ---
.../org/apache/iceberg/flink/TypeToFlinkType.java | 134 ---
.../org/apache/iceberg/flink/actions/Actions.java | 52 -
.../flink/actions/RewriteDataFilesAction.java | 72 --
.../flink/data/AvroWithFlinkSchemaVisitor.java | 75 --
.../apache/iceberg/flink/data/FlinkAvroReader.java | 169 ---
.../apache/iceberg/flink/data/FlinkAvroWriter.java | 165 ---
.../apache/iceberg/flink/data/FlinkOrcReader.java | 131 ---
.../apache/iceberg/flink/data/FlinkOrcReaders.java | 283 -----
.../apache/iceberg/flink/data/FlinkOrcWriter.java | 163 ---
.../apache/iceberg/flink/data/FlinkOrcWriters.java | 317 ------
.../iceberg/flink/data/FlinkParquetReaders.java | 832 --------------
.../iceberg/flink/data/FlinkParquetWriters.java | 504 ---------
.../iceberg/flink/data/FlinkSchemaVisitor.java | 161 ---
.../iceberg/flink/data/FlinkValueReaders.java | 312 ------
.../iceberg/flink/data/FlinkValueWriters.java | 253 -----
.../flink/data/ParquetWithFlinkSchemaVisitor.java | 222 ----
.../iceberg/flink/data/RowDataProjection.java | 341 ------
.../org/apache/iceberg/flink/data/RowDataUtil.java | 100 --
.../apache/iceberg/flink/data/StructRowData.java | 300 -----
.../sink/AvroGenericRecordToRowDataMapper.java | 61 -
.../iceberg/flink/sink/BaseDeltaTaskWriter.java | 125 ---
.../flink/sink/BucketPartitionKeySelector.java | 70 --
.../iceberg/flink/sink/BucketPartitioner.java | 103 --
.../iceberg/flink/sink/BucketPartitionerUtil.java | 125 ---
.../iceberg/flink/sink/CachingTableSupplier.java | 91 --
.../apache/iceberg/flink/sink/CommitSummary.java | 93 --
.../apache/iceberg/flink/sink/DeltaManifests.java | 71 --
.../flink/sink/DeltaManifestsSerializer.java | 122 --
.../flink/sink/EqualityFieldKeySelector.java | 86 --
.../iceberg/flink/sink/FlinkAppenderFactory.java | 274 -----
.../iceberg/flink/sink/FlinkFileWriterFactory.java | 293 -----
.../iceberg/flink/sink/FlinkManifestUtil.java | 132 ---
.../org/apache/iceberg/flink/sink/FlinkSink.java | 654 -----------
.../iceberg/flink/sink/IcebergFilesCommitter.java | 516 ---------
.../flink/sink/IcebergFilesCommitterMetrics.java | 96 --
.../iceberg/flink/sink/IcebergStreamWriter.java | 120 --
.../flink/sink/IcebergStreamWriterMetrics.java | 89 --
.../flink/sink/ManifestOutputFileFactory.java | 94 --
.../iceberg/flink/sink/PartitionKeySelector.java | 64 --
.../iceberg/flink/sink/PartitionedDeltaWriter.java | 97 --
.../flink/sink/RowDataTaskWriterFactory.java | 244 ----
.../iceberg/flink/sink/TaskWriterFactory.java | 45 -
.../flink/sink/UnpartitionedDeltaWriter.java | 69 --
.../flink/sink/shuffle/AggregatedStatistics.java | 71 --
.../sink/shuffle/AggregatedStatisticsTracker.java | 133 ---
.../iceberg/flink/sink/shuffle/DataStatistics.java | 57 -
.../sink/shuffle/DataStatisticsCoordinator.java | 396 -------
.../shuffle/DataStatisticsCoordinatorProvider.java | 51 -
.../flink/sink/shuffle/DataStatisticsEvent.java | 57 -
.../flink/sink/shuffle/DataStatisticsOperator.java | 197 ----
.../flink/sink/shuffle/DataStatisticsOrRecord.java | 113 --
.../shuffle/DataStatisticsOrRecordSerializer.java | 219 ----
.../flink/sink/shuffle/DataStatisticsUtil.java | 96 --
.../flink/sink/shuffle/MapDataStatistics.java | 70 --
.../sink/shuffle/MapDataStatisticsSerializer.java | 187 ----
.../flink/sink/shuffle/MapRangePartitioner.java | 381 -------
.../flink/sink/shuffle/SortKeySerializer.java | 353 ------
.../AvroGenericRecordFileScanTaskReader.java | 42 -
.../apache/iceberg/flink/source/DataIterator.java | 156 ---
.../iceberg/flink/source/DataTaskReader.java | 47 -
.../iceberg/flink/source/FileScanTaskReader.java | 35 -
.../iceberg/flink/source/FlinkInputFormat.java | 141 ---
.../iceberg/flink/source/FlinkInputSplit.java | 48 -
.../apache/iceberg/flink/source/FlinkSource.java | 310 ------
.../iceberg/flink/source/FlinkSplitPlanner.java | 189 ----
.../apache/iceberg/flink/source/IcebergSource.java | 543 ---------
.../iceberg/flink/source/IcebergTableSource.java | 229 ----
.../flink/source/RowDataFileScanTaskReader.java | 243 ----
.../iceberg/flink/source/RowDataRewriter.java | 172 ---
.../RowDataToAvroGenericRecordConverter.java | 70 --
.../apache/iceberg/flink/source/ScanContext.java | 592 ----------
.../apache/iceberg/flink/source/SourceUtil.java | 77 --
.../flink/source/StreamingMonitorFunction.java | 269 -----
.../flink/source/StreamingReaderOperator.java | 246 ----
.../flink/source/StreamingStartingStrategy.java | 54 -
.../source/assigner/DefaultSplitAssigner.java | 112 --
.../flink/source/assigner/GetSplitResult.java | 77 --
.../assigner/OrderedSplitAssignerFactory.java | 46 -
.../assigner/SimpleSplitAssignerFactory.java | 37 -
.../flink/source/assigner/SplitAssigner.java | 118 --
.../source/assigner/SplitAssignerFactory.java | 30 -
.../flink/source/assigner/SplitAssignerType.java | 33 -
.../enumerator/AbstractIcebergEnumerator.java | 173 ---
.../enumerator/ContinuousEnumerationResult.java | 57 -
.../enumerator/ContinuousIcebergEnumerator.java | 178 ---
.../source/enumerator/ContinuousSplitPlanner.java | 30 -
.../enumerator/ContinuousSplitPlannerImpl.java | 240 ----
.../source/enumerator/EnumerationHistory.java | 96 --
.../enumerator/IcebergEnumeratorPosition.java | 79 --
.../IcebergEnumeratorPositionSerializer.java | 90 --
.../source/enumerator/IcebergEnumeratorState.java | 65 --
.../IcebergEnumeratorStateSerializer.java | 196 ----
.../source/enumerator/StaticIcebergEnumerator.java | 51 -
.../flink/source/reader/ArrayBatchRecords.java | 171 ---
.../reader/ArrayPoolDataIteratorBatcher.java | 130 ---
.../reader/AvroGenericRecordReaderFunction.java | 102 --
.../reader/ColumnStatsWatermarkExtractor.java | 98 --
.../flink/source/reader/DataIteratorBatcher.java | 36 -
.../source/reader/DataIteratorReaderFunction.java | 43 -
.../flink/source/reader/IcebergSourceReader.java | 77 --
.../source/reader/IcebergSourceReaderMetrics.java | 61 -
.../source/reader/IcebergSourceSplitReader.java | 156 ---
.../flink/source/reader/ListBatchRecords.java | 85 --
.../source/reader/ListDataIteratorBatcher.java | 94 --
.../source/reader/MetaDataReaderFunction.java | 65 --
.../flink/source/reader/ReaderFunction.java | 31 -
.../flink/source/reader/RecordAndPosition.java | 78 --
.../iceberg/flink/source/reader/RecordFactory.java | 34 -
.../flink/source/reader/RowDataReaderFunction.java | 79 --
.../flink/source/reader/RowDataRecordFactory.java | 61 -
.../source/reader/SerializableRecordEmitter.java | 40 -
.../source/reader/SplitWatermarkExtractor.java | 28 -
.../reader/WatermarkExtractorRecordEmitter.java | 67 --
.../flink/source/split/IcebergSourceSplit.java | 177 ---
.../source/split/IcebergSourceSplitSerializer.java | 60 -
.../source/split/IcebergSourceSplitState.java | 37 -
.../source/split/IcebergSourceSplitStatus.java | 25 -
.../flink/source/split/SerializableComparator.java | 24 -
.../flink/source/split/SplitComparators.java | 76 --
.../flink/source/split/SplitRequestEvent.java | 54 -
.../iceberg/flink/util/FlinkCompatibilityUtil.java | 42 -
.../apache/iceberg/flink/util/FlinkPackage.java | 61 -
.../org.apache.flink.table.factories.Factory | 16 -
.../org.apache.flink.table.factories.TableFactory | 16 -
.../flink/AvroGenericRecordConverterBase.java | 90 --
.../org/apache/iceberg/flink/CatalogTestBase.java | 143 ---
.../org/apache/iceberg/flink/DataGenerator.java | 42 -
.../org/apache/iceberg/flink/DataGenerators.java | 1172 --------------------
.../org/apache/iceberg/flink/FlinkTestBase.java | 129 ---
.../iceberg/flink/HadoopCatalogExtension.java | 104 --
.../iceberg/flink/HadoopCatalogResource.java | 90 --
.../apache/iceberg/flink/HadoopTableResource.java | 64 --
.../apache/iceberg/flink/MiniClusterResource.java | 53 -
.../iceberg/flink/MiniFlinkClusterExtension.java | 53 -
.../org/apache/iceberg/flink/RowDataConverter.java | 135 ---
.../org/apache/iceberg/flink/SimpleDataUtil.java | 443 --------
.../java/org/apache/iceberg/flink/TestBase.java | 130 ---
.../apache/iceberg/flink/TestCatalogLoader.java | 118 --
.../iceberg/flink/TestCatalogTableLoader.java | 116 --
.../apache/iceberg/flink/TestChangeLogTable.java | 305 -----
.../iceberg/flink/TestDataFileSerialization.java | 207 ----
.../org/apache/iceberg/flink/TestFixtures.java | 59 -
.../iceberg/flink/TestFlinkAnonymousTable.java | 64 --
.../iceberg/flink/TestFlinkCatalogDatabase.java | 254 -----
.../iceberg/flink/TestFlinkCatalogFactory.java | 115 --
.../iceberg/flink/TestFlinkCatalogTable.java | 413 -------
.../flink/TestFlinkCatalogTablePartitions.java | 118 --
.../apache/iceberg/flink/TestFlinkConfParser.java | 61 -
.../org/apache/iceberg/flink/TestFlinkFilters.java | 467 --------
.../apache/iceberg/flink/TestFlinkHiveCatalog.java | 103 --
.../apache/iceberg/flink/TestFlinkSchemaUtil.java | 415 -------
.../apache/iceberg/flink/TestFlinkTableSink.java | 358 ------
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 334 ------
.../java/org/apache/iceberg/flink/TestHelpers.java | 628 -----------
.../apache/iceberg/flink/TestIcebergConnector.java | 350 ------
.../flink/TestManifestFileSerialization.java | 174 ---
.../apache/iceberg/flink/TestRowDataWrapper.java | 93 --
.../org/apache/iceberg/flink/TestTableLoader.java | 57 -
.../iceberg/flink/TestTableSerialization.java | 110 --
.../flink/actions/TestRewriteDataFilesAction.java | 481 --------
.../apache/iceberg/flink/data/RandomRowData.java | 38 -
.../iceberg/flink/data/RowDataToRowMapper.java | 50 -
.../flink/data/TestFlinkAvroReaderWriter.java | 185 ---
.../flink/data/TestFlinkOrcReaderWriter.java | 107 --
.../iceberg/flink/data/TestFlinkParquetReader.java | 138 ---
.../iceberg/flink/data/TestFlinkParquetWriter.java | 94 --
.../iceberg/flink/data/TestRowDataProjection.java | 593 ----------
.../iceberg/flink/data/TestRowProjection.java | 580 ----------
.../iceberg/flink/data/TestStructRowData.java | 100 --
.../sink/TestAvroGenericRecordToRowDataMapper.java | 37 -
.../flink/sink/TestBucketPartitionKeySelector.java | 65 --
.../iceberg/flink/sink/TestBucketPartitioner.java | 107 --
.../TestBucketPartitionerFlinkIcebergSink.java | 227 ----
.../flink/sink/TestBucketPartitionerUtil.java | 126 ---
.../flink/sink/TestCachingTableSupplier.java | 81 --
.../flink/sink/TestCompressionSettings.java | 254 -----
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 429 -------
.../flink/sink/TestFlinkAppenderFactory.java | 65 --
.../flink/sink/TestFlinkFileWriterFactory.java | 66 --
.../iceberg/flink/sink/TestFlinkIcebergSink.java | 397 -------
.../flink/sink/TestFlinkIcebergSinkBase.java | 64 --
.../flink/sink/TestFlinkIcebergSinkBranch.java | 137 ---
.../iceberg/flink/sink/TestFlinkIcebergSinkV2.java | 236 ----
.../flink/sink/TestFlinkIcebergSinkV2Base.java | 361 ------
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 134 ---
.../iceberg/flink/sink/TestFlinkManifest.java | 315 ------
.../flink/sink/TestFlinkPartitioningWriters.java | 66 --
.../flink/sink/TestFlinkPositionDeltaWriters.java | 66 --
.../flink/sink/TestFlinkRollingFileWriters.java | 51 -
.../iceberg/flink/sink/TestFlinkWriterMetrics.java | 60 -
.../flink/sink/TestIcebergFilesCommitter.java | 1148 -------------------
.../flink/sink/TestIcebergStreamWriter.java | 388 -------
.../flink/sink/TestRowDataPartitionKey.java | 252 -----
.../apache/iceberg/flink/sink/TestTaskWriters.java | 243 ----
.../sink/shuffle/TestAggregatedStatistics.java | 63 --
.../shuffle/TestAggregatedStatisticsTracker.java | 178 ---
.../shuffle/TestDataStatisticsCoordinator.java | 178 ---
.../TestDataStatisticsCoordinatorProvider.java | 143 ---
.../sink/shuffle/TestDataStatisticsOperator.java | 255 -----
.../flink/sink/shuffle/TestMapDataStatistics.java | 90 --
.../sink/shuffle/TestMapRangePartitioner.java | 448 --------
.../sink/shuffle/TestSortKeySerializerBase.java | 65 --
.../shuffle/TestSortKeySerializerNestedStruct.java | 55 -
.../shuffle/TestSortKeySerializerPrimitives.java | 57 -
.../iceberg/flink/source/BoundedTableFactory.java | 170 ---
.../iceberg/flink/source/BoundedTestSource.java | 108 --
.../flink/source/ChangeLogTableTestBase.java | 87 --
.../apache/iceberg/flink/source/SplitHelpers.java | 132 ---
.../apache/iceberg/flink/source/SqlHelpers.java | 60 -
.../flink/source/TestBoundedTableFactory.java | 87 --
.../iceberg/flink/source/TestFlinkInputFormat.java | 191 ----
.../source/TestFlinkInputFormatReaderDeletes.java | 70 --
.../flink/source/TestFlinkMergingMetrics.java | 71 --
.../flink/source/TestFlinkMetaDataTable.java | 813 --------------
.../flink/source/TestFlinkReaderDeletesBase.java | 89 --
.../apache/iceberg/flink/source/TestFlinkScan.java | 540 ---------
.../iceberg/flink/source/TestFlinkScanSql.java | 69 --
.../iceberg/flink/source/TestFlinkSource.java | 90 --
.../flink/source/TestFlinkSourceConfig.java | 58 -
.../iceberg/flink/source/TestFlinkSourceSql.java | 85 --
.../iceberg/flink/source/TestFlinkTableSource.java | 607 ----------
.../flink/source/TestIcebergSourceBounded.java | 127 ---
.../TestIcebergSourceBoundedGenericRecord.java | 203 ----
.../flink/source/TestIcebergSourceBoundedSql.java | 76 --
.../flink/source/TestIcebergSourceContinuous.java | 508 ---------
.../flink/source/TestIcebergSourceFailover.java | 297 -----
...cebergSourceFailoverWithWatermarkExtractor.java | 112 --
.../source/TestIcebergSourceReaderDeletes.java | 102 --
.../iceberg/flink/source/TestIcebergSourceSql.java | 158 ---
.../TestIcebergSourceWithWatermarkExtractor.java | 408 -------
.../source/TestMetadataTableReadableMetrics.java | 299 -----
.../flink/source/TestProjectMetaColumn.java | 186 ----
.../TestRowDataToAvroGenericRecordConverter.java | 35 -
.../iceberg/flink/source/TestSourceUtil.java | 60 -
.../apache/iceberg/flink/source/TestSqlBase.java | 158 ---
.../iceberg/flink/source/TestStreamScanSql.java | 430 -------
.../flink/source/TestStreamingMonitorFunction.java | 401 -------
.../flink/source/TestStreamingReaderOperator.java | 293 -----
.../source/assigner/SplitAssignerTestBase.java | 130 ---
.../source/assigner/TestDefaultSplitAssigner.java | 43 -
.../TestFileSequenceNumberBasedSplitAssigner.java | 80 --
.../assigner/TestWatermarkBasedSplitAssigner.java | 146 ---
.../enumerator/ManualContinuousSplitPlanner.java | 97 --
.../TestContinuousIcebergEnumerator.java | 352 ------
.../enumerator/TestContinuousSplitPlannerImpl.java | 688 ------------
...estContinuousSplitPlannerImplStartStrategy.java | 205 ----
.../source/enumerator/TestEnumerationHistory.java | 134 ---
.../TestIcebergEnumeratorStateSerializer.java | 145 ---
.../source/reader/ReaderFunctionTestBase.java | 219 ----
.../iceberg/flink/source/reader/ReaderUtil.java | 125 ---
.../flink/source/reader/TestArrayBatchRecords.java | 68 --
.../TestArrayPoolDataIteratorBatcherRowData.java | 358 ------
.../reader/TestColumnStatsWatermarkExtractor.java | 178 ---
.../source/reader/TestIcebergSourceReader.java | 184 ---
.../source/reader/TestRowDataReaderFunction.java | 74 --
.../flink/source/reader/TestingMetricGroup.java | 102 --
.../split/TestIcebergSourceSplitSerializer.java | 162 ---
.../iceberg/flink/util/TestFlinkPackage.java | 54 -
.../org.apache.flink.table.factories.Factory | 16 -
284 files changed, 50366 deletions(-)
diff --git a/flink/v1.16/build.gradle b/flink/v1.16/build.gradle
deleted file mode 100644
index 335a471e24..0000000000
--- a/flink/v1.16/build.gradle
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-String flinkMajorVersion = '1.16'
-String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
-
-project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
-
- dependencies {
- implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
- api project(':iceberg-api')
- implementation project(':iceberg-common')
- implementation project(':iceberg-core')
- api project(':iceberg-data')
- implementation project(':iceberg-orc')
- implementation project(':iceberg-parquet')
- implementation project(':iceberg-hive-metastore')
-
- compileOnly libs.flink116.avro
- // for dropwizard histogram metrics implementation
- compileOnly libs.flink116.metrics.dropwizard
- compileOnly libs.flink116.streaming.java
- compileOnly "${libs.flink116.streaming.java.get().module}:${libs.flink116.streaming.java.get().getVersion()}:tests"
- compileOnly libs.flink116.table.api.java.bridge
- compileOnly "org.apache.flink:flink-table-planner_${scalaVersion}:${libs.versions.flink116.get()}"
- compileOnly libs.flink116.connector.base
- compileOnly libs.flink116.connector.files
-
- compileOnly libs.hadoop2.hdfs
- compileOnly libs.hadoop2.common
- compileOnly(libs.hadoop2.minicluster) {
- exclude group: 'org.apache.avro', module: 'avro'
- }
-
- implementation(libs.parquet.avro) {
- exclude group: 'org.apache.avro', module: 'avro'
- // already shaded by Parquet
- exclude group: 'it.unimi.dsi'
- exclude group: 'org.codehaus.jackson'
- }
-
- compileOnly libs.avro.avro
-
- implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
- exclude group: 'org.apache.hadoop'
- exclude group: 'commons-lang'
- // These artifacts are shaded and included in the orc-core fat jar
- exclude group: 'com.google.protobuf', module: 'protobuf-java'
- exclude group: 'org.apache.hive', module: 'hive-storage-api'
- exclude group: 'org.slf4j'
- }
-
- testImplementation libs.flink116.connector.test.utils
- testImplementation libs.flink116.core
- testImplementation libs.flink116.runtime
- testImplementation (libs.flink116.test.utilsjunit) {
- exclude group: 'junit'
- }
- testImplementation(libs.flink116.test.utils) {
- exclude group: "org.apache.curator", module: 'curator-test'
- exclude group: 'junit'
- }
-
- testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
-
- // By default, hive-exec is a fat/uber jar and it exports a guava library
- // that's really old. We use the core classifier to be able to override our guava
- // version. Luckily, hive-exec seems to work okay so far with this version of guava
- // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
- testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.slf4j', module: 'slf4j-log4j12'
- exclude group: 'org.pentaho' // missing dependency
- exclude group: 'org.apache.hive', module: 'hive-llap-tez'
- exclude group: 'org.apache.logging.log4j'
- exclude group: 'com.google.protobuf', module: 'protobuf-java'
- exclude group: 'org.apache.calcite'
- exclude group: 'org.apache.calcite.avatica'
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
-
- testImplementation(libs.hive2.metastore) {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.slf4j', module: 'slf4j-log4j12'
- exclude group: 'org.pentaho' // missing dependency
- exclude group: 'org.apache.hbase'
- exclude group: 'org.apache.logging.log4j'
- exclude group: 'co.cask.tephra'
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
- exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
- exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
- exclude group: 'com.tdunning', module: 'json'
- exclude group: 'javax.transaction', module: 'transaction-api'
- exclude group: 'com.zaxxer', module: 'HikariCP'
- exclude group: 'org.slf4j'
- }
-
- testImplementation libs.awaitility
- testImplementation libs.assertj.core
- }
-
- test {
- useJUnitPlatform()
- }
-}
-
-project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
- apply plugin: 'com.github.johnrengelman.shadow'
-
- tasks.jar.dependsOn tasks.shadowJar
-
- sourceSets {
- integration {
- java.srcDir "$projectDir/src/integration/java"
- resources.srcDir "$projectDir/src/integration/resources"
- }
- }
-
- configurations {
- implementation {
- // included in Flink
- exclude group: 'org.slf4j'
- exclude group: 'org.apache.commons'
- exclude group: 'commons-pool'
- exclude group: 'commons-codec'
- exclude group: 'org.xerial.snappy'
- exclude group: 'javax.xml.bind'
- exclude group: 'javax.annotation'
- }
- }
-
- dependencies {
- implementation(project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}")) {
- exclude group: 'org.apache.flink'
- }
- implementation project(':iceberg-aws')
- implementation project(':iceberg-azure')
- implementation(project(':iceberg-aliyun')) {
- exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
- exclude group: 'org.apache.httpcomponents', module: 'httpclient'
- exclude group: 'commons-logging', module: 'commons-logging'
- }
- implementation project(':iceberg-gcp')
- implementation(project(':iceberg-nessie')) {
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
-
- // for dropwizard histogram metrics implementation
- implementation libs.flink116.metrics.dropwizard
-
- // for integration testing with the flink-runtime-jar
- // all of those dependencies are required because the integration test extends FlinkTestBase
- integrationCompileOnly project(':iceberg-api')
- integrationImplementation libs.junit.vintage.engine
- integrationImplementation libs.assertj.core
- integrationImplementation project(path: ":iceberg-flink:iceberg-flink-${flinkMajorVersion}", configuration: "testArtifacts")
- integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- integrationImplementation(libs.flink116.test.utils) {
- exclude group: "org.apache.curator", module: 'curator-test'
- exclude group: 'junit'
- }
-
- integrationImplementation libs.flink116.table.api.java.bridge
- integrationImplementation "org.apache.flink:flink-table-planner_${scalaVersion}:${libs.versions.flink116.get()}"
-
- integrationImplementation libs.hadoop2.common
- integrationImplementation libs.hadoop2.hdfs
- integrationImplementation(libs.hadoop2.minicluster) {
- exclude group: 'org.apache.avro', module: 'avro'
- }
-
- integrationImplementation(libs.hive2.metastore) {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.slf4j', module: 'slf4j-log4j12'
- exclude group: 'org.pentaho' // missing dependency
- exclude group: 'org.apache.hbase'
- exclude group: 'org.apache.logging.log4j'
- exclude group: 'co.cask.tephra'
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
- exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
- exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
- exclude group: 'com.tdunning', module: 'json'
- exclude group: 'javax.transaction', module: 'transaction-api'
- exclude group: 'com.zaxxer', module: 'HikariCP'
- exclude group: 'org.slf4j'
- }
-
- integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.slf4j', module: 'slf4j-log4j12'
- exclude group: 'org.pentaho' // missing dependency
- exclude group: 'org.apache.hive', module: 'hive-llap-tez'
- exclude group: 'org.apache.logging.log4j'
- exclude group: 'com.google.protobuf', module: 'protobuf-java'
- exclude group: 'org.apache.calcite'
- exclude group: 'org.apache.calcite.avatica'
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
- }
-
- shadowJar {
- configurations = [project.configurations.runtimeClasspath]
-
- zip64 true
-
- // include the LICENSE and NOTICE files for the shaded Jar
- from(projectDir) {
- include 'LICENSE'
- include 'NOTICE'
- }
-
- // Relocate dependencies to avoid conflicts
- relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
- relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
- relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
- relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
- relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
- relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
- relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
- relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
- relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
- relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
- relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
- relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
- relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
-
- archiveClassifier.set(null)
- }
-
- task integrationTest(type: Test) {
- description = "Test Flink Runtime Jar against Flink ${flinkMajorVersion}"
- group = "verification"
- jvmArgs += project.property('extraJvmArgs')
- testClassesDirs = sourceSets.integration.output.classesDirs
- classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
- inputs.file(shadowJar.archiveFile.get().asFile.path)
- }
- integrationTest.dependsOn shadowJar
- check.dependsOn integrationTest
-
- jar {
- enabled = false
- }
-}
diff --git a/flink/v1.16/flink-runtime/LICENSE b/flink/v1.16/flink-runtime/LICENSE
deleted file mode 100644
index 8ab53469eb..0000000000
--- a/flink/v1.16/flink-runtime/LICENSE
+++ /dev/null
@@ -1,502 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Apache Avro.
-
-Copyright: 2014-2020 The Apache Software Foundation.
-Home page: https://parquet.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains the Jackson JSON processor.
-
-Copyright: 2007-2020 Tatu Saloranta and other contributors
-Home page: http://jackson.codehaus.org/
-License: http://www.apache.org/licenses/LICENSE-2.0.txt
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Apache Parquet.
-
-Copyright: 2014-2020 The Apache Software Foundation.
-Home page: https://parquet.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Apache Thrift.
-
-Copyright: 2006-2010 The Apache Software Foundation.
-Home page: https://thrift.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains fastutil.
-
-Copyright: 2002-2014 Sebastiano Vigna
-Home page: http://fastutil.di.unimi.it/
-License: http://www.apache.org/licenses/LICENSE-2.0.html
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Apache ORC.
-
-Copyright: 2013-2020 The Apache Software Foundation.
-Home page: https://orc.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Apache Hive's storage API via ORC.
-
-Copyright: 2013-2020 The Apache Software Foundation.
-Home page: https://hive.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Airlift Aircompressor.
-
-Copyright: 2011-2020 Aircompressor authors.
-Home page: https://github.com/airlift/aircompressor
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Airlift Slice.
-
-Copyright: 2013-2020 Slice authors.
-Home page: https://github.com/airlift/slice
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains JetBrains annotations.
-
-Copyright: 2000-2020 JetBrains s.r.o.
-Home page: https://github.com/JetBrains/java-annotations
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Google Guava.
-
-Copyright: 2006-2020 The Guava Authors
-Home page: https://github.com/google/guava
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Google Error Prone Annotations.
-
-Copyright: Copyright 2011-2019 The Error Prone Authors
-Home page: https://github.com/google/error-prone
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains checkerframework checker-qual Annotations.
-
-Copyright: 2004-2020 the Checker Framework developers
-Home page: https://github.com/typetools/checker-framework
-License: https://github.com/typetools/checker-framework/blob/master/LICENSE.txt (MIT license)
-
-License text:
-| The annotations are licensed under the MIT License. (The text of this
-| license appears below.) More specifically, all the parts of the Checker
-| Framework that you might want to include with your own program use the
-| MIT License. This is the checker-qual.jar file and all the files that
-| appear in it: every file in a qual/ directory, plus utility files such
-| as NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc.
-| In addition, the cleanroom implementations of third-party annotations,
-| which the Checker Framework recognizes as aliases for its own
-| annotations, are licensed under the MIT License.
-|
-| Permission is hereby granted, free of charge, to any person obtaining a copy
-| of this software and associated documentation files (the "Software"), to deal
-| in the Software without restriction, including without limitation the rights
-| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-| copies of the Software, and to permit persons to whom the Software is
-| furnished to do so, subject to the following conditions:
-|
-| The above copyright notice and this permission notice shall be included in
-| all copies or substantial portions of the Software.
-|
-| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-| THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Animal Sniffer Annotations.
-
-Copyright: 2009-2018 codehaus.org
-Home page: https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations/
-License: https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations/license.html (MIT license)
-
-License text:
-| The MIT License
-|
-| Copyright (c) 2009 codehaus.org.
-|
-| Permission is hereby granted, free of charge, to any person obtaining a copy
-| of this software and associated documentation files (the "Software"), to deal
-| in the Software without restriction, including without limitation the rights
-| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-| copies of the Software, and to permit persons to whom the Software is
-| furnished to do so, subject to the following conditions:
-|
-| The above copyright notice and this permission notice shall be included in
-| all copies or substantial portions of the Software.
-|
-| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-| THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Caffeine by Ben Manes.
-
-Copyright: 2014-2020 Ben Manes and contributors
-Home page: https://github.com/ben-manes/caffeine
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Apache Yetus audience annotations.
-
-Copyright: 2008-2020 The Apache Software Foundation.
-Home page: https://yetus.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Google protobuf.
-
-Copyright: 2008 Google Inc.
-Home page: https://developers.google.com/protocol-buffers
-License: https://github.com/protocolbuffers/protobuf/blob/master/LICENSE (BSD)
-
-License text:
-
-| Copyright 2008 Google Inc. All rights reserved.
-|
-| Redistribution and use in source and binary forms, with or without
-| modification, are permitted provided that the following conditions are
-| met:
-|
-| * Redistributions of source code must retain the above copyright
-| notice, this list of conditions and the following disclaimer.
-| * Redistributions in binary form must reproduce the above
-| copyright notice, this list of conditions and the following disclaimer
-| in the documentation and/or other materials provided with the
-| distribution.
-| * Neither the name of Google Inc. nor the names of its
-| contributors may be used to endorse or promote products derived from
-| this software without specific prior written permission.
-|
-| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-|
-| Code generated by the Protocol Buffer compiler is owned by the owner
-| of the input file used when generating it. This code is not
-| standalone and requires a support library to be linked with it. This
-| support library is itself covered by the above license.
-
---------------------------------------------------------------------------------
-
-This binary artifact contains ThreeTen.
-
-Copyright: 2007-present, Stephen Colebourne & Michael Nascimento Santos.
-Home page: https://www.threeten.org/threeten-extra/
-License: https://github.com/ThreeTen/threeten-extra/blob/master/LICENSE.txt (BSD 3-clause)
-
-License text:
-
-| All rights reserved.
-|
-| * Redistribution and use in source and binary forms, with or without
-| modification, are permitted provided that the following conditions are met:
-|
-| * Redistributions of source code must retain the above copyright notice,
-| this list of conditions and the following disclaimer.
-|
-| * Redistributions in binary form must reproduce the above copyright notice,
-| this list of conditions and the following disclaimer in the documentation
-| and/or other materials provided with the distribution.
-|
-| * Neither the name of JSR-310 nor the names of its contributors
-| may be used to endorse or promote products derived from this software
-| without specific prior written permission.
-|
-| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-| CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This binary artifact includes Project Nessie with the following in its NOTICE
-file:
-
-| Dremio
-| Copyright 2015-2017 Dremio Corporation
-|
-| This product includes software developed at
-| The Apache Software Foundation (http://www.apache.org/).
-
---------------------------------------------------------------------------------
-
-This binary includes code from Apache Commons.
-
-* Core ArrayUtil.
-
-Copyright: 2020 The Apache Software Foundation
-Home page: https://commons.apache.org/
-License: https://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This binary artifact contains Apache HttpComponents Client.
-
-Copyright: 1999-2022 The Apache Software Foundation.
-Home page: https://hc.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This product includes code from Apache HttpComponents Client.
-
-* retry and error handling logic in ExponentialHttpRequestRetryStrategy.java
-
-Copyright: 1999-2022 The Apache Software Foundation.
-Home page: https://hc.apache.org/
-License: https://www.apache.org/licenses/LICENSE-2.0
diff --git a/flink/v1.16/flink-runtime/NOTICE b/flink/v1.16/flink-runtime/NOTICE
deleted file mode 100644
index 81aa166045..0000000000
--- a/flink/v1.16/flink-runtime/NOTICE
+++ /dev/null
@@ -1,91 +0,0 @@
-
-Apache Iceberg
-Copyright 2017-2022 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
---------------------------------------------------------------------------------
-
-This binary artifact includes Apache ORC with the following in its NOTICE file:
-
-| Apache ORC
-| Copyright 2013-2019 The Apache Software Foundation
-|
-| This product includes software developed by The Apache Software
-| Foundation (http://www.apache.org/).
-|
-| This product includes software developed by Hewlett-Packard:
-| (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
-
---------------------------------------------------------------------------------
-
-This binary artifact includes Airlift Aircompressor with the following in its
-NOTICE file:
-
-| Snappy Copyright Notices
-| =========================
-|
-| * Copyright 2011 Dain Sundstrom <da...@iq80.com>
-| * Copyright 2011, Google Inc.<op...@google.com>
-|
-|
-| Snappy License
-| ===============
-| Copyright 2011, Google Inc.
-| All rights reserved.
-|
-| Redistribution and use in source and binary forms, with or without
-| modification, are permitted provided that the following conditions are
-| met:
-|
-| * Redistributions of source code must retain the above copyright
-| notice, this list of conditions and the following disclaimer.
-| * Redistributions in binary form must reproduce the above
-| copyright notice, this list of conditions and the following disclaimer
-| in the documentation and/or other materials provided with the
-| distribution.
-| * Neither the name of Google Inc. nor the names of its
-| contributors may be used to endorse or promote products derived from
-| this software without specific prior written permission.
-|
-| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This binary artifact includes Apache Yetus with the following in its NOTICE
-file:
-
-| Apache Yetus
-| Copyright 2008-2020 The Apache Software Foundation
-|
-| This product includes software developed at
-| The Apache Software Foundation (https://www.apache.org/).
-|
-| ---
-| Additional licenses for the Apache Yetus Source/Website:
-| ---
-|
-|
-| See LICENSE for terms.
-
---------------------------------------------------------------------------------
-
-This binary artifact includes Project Nessie with the following in its NOTICE
-file:
-
-| Dremio
-| Copyright 2015-2017 Dremio Corporation
-|
-| This product includes software developed at
-| The Apache Software Foundation (http://www.apache.org/).
diff --git a/flink/v1.16/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java b/flink/v1.16/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java
deleted file mode 100644
index 65cfba1ec8..0000000000
--- a/flink/v1.16/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.util.Map;
-
-public class IcebergConnectorSmokeTest extends TestIcebergConnector {
-
- public IcebergConnectorSmokeTest(
- String catalogName, Map<String, String> properties, boolean isStreaming) {
- super(catalogName, properties, isStreaming);
- }
-}
diff --git a/flink/v1.16/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java b/flink/v1.16/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java
deleted file mode 100644
index c391716575..0000000000
--- a/flink/v1.16/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.sink.shuffle;
-
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.RowData;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.SortKey;
-import org.apache.iceberg.SortOrder;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.types.Types;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
-
-@Fork(1)
-@State(Scope.Benchmark)
-@Warmup(iterations = 3)
-@Measurement(iterations = 5)
-@BenchmarkMode(Mode.SingleShotTime)
-public class MapRangePartitionerBenchmark {
- private static final String CHARS =
- "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.!?";
- private static final int SAMPLE_SIZE = 100_000;
- private static final Schema SCHEMA =
- new Schema(
- Types.NestedField.required(1, "id", Types.IntegerType.get()),
- Types.NestedField.required(2, "name2", Types.StringType.get()),
- Types.NestedField.required(3, "name3", Types.StringType.get()),
- Types.NestedField.required(4, "name4", Types.StringType.get()),
- Types.NestedField.required(5, "name5", Types.StringType.get()),
- Types.NestedField.required(6, "name6", Types.StringType.get()),
- Types.NestedField.required(7, "name7", Types.StringType.get()),
- Types.NestedField.required(8, "name8", Types.StringType.get()),
- Types.NestedField.required(9, "name9", Types.StringType.get()));
-
- private static final SortOrder SORT_ORDER = SortOrder.builderFor(SCHEMA).asc("id").build();
- private static final SortKey SORT_KEY = new SortKey(SCHEMA, SORT_ORDER);
-
- private MapRangePartitioner partitioner;
- private RowData[] rows;
-
- @Setup
- public void setupBenchmark() {
- NavigableMap<Integer, Long> weights = longTailDistribution(100_000, 24, 240, 100, 2.0);
- Map<SortKey, Long> mapStatistics = Maps.newHashMapWithExpectedSize(weights.size());
- weights.forEach(
- (id, weight) -> {
- SortKey sortKey = SORT_KEY.copy();
- sortKey.set(0, id);
- mapStatistics.put(sortKey, weight);
- });
-
- MapDataStatistics dataStatistics = new MapDataStatistics(mapStatistics);
- this.partitioner =
- new MapRangePartitioner(
- SCHEMA, SortOrder.builderFor(SCHEMA).asc("id").build(), dataStatistics, 2);
-
- List<Integer> keys = Lists.newArrayList(weights.keySet().iterator());
- long[] weightsCDF = new long[keys.size()];
- long totalWeight = 0;
- for (int i = 0; i < keys.size(); ++i) {
- totalWeight += weights.get(keys.get(i));
- weightsCDF[i] = totalWeight;
- }
-
- // pre-calculate the samples for benchmark run
- this.rows = new GenericRowData[SAMPLE_SIZE];
- for (int i = 0; i < SAMPLE_SIZE; ++i) {
- long weight = ThreadLocalRandom.current().nextLong(totalWeight);
- int index = binarySearchIndex(weightsCDF, weight);
- rows[i] =
- GenericRowData.of(
- keys.get(index),
- randomString("name2-"),
- randomString("name3-"),
- randomString("name4-"),
- randomString("name5-"),
- randomString("name6-"),
- randomString("name7-"),
- randomString("name8-"),
- randomString("name9-"));
- }
- }
-
- @TearDown
- public void tearDownBenchmark() {}
-
- @Benchmark
- @Threads(1)
- public void testPartitionerLongTailDistribution(Blackhole blackhole) {
- for (int i = 0; i < SAMPLE_SIZE; ++i) {
- blackhole.consume(partitioner.partition(rows[i], 128));
- }
- }
-
- private static String randomString(String prefix) {
- int length = ThreadLocalRandom.current().nextInt(200);
- byte[] buffer = new byte[length];
-
- for (int i = 0; i < length; i += 1) {
- buffer[i] = (byte) CHARS.charAt(ThreadLocalRandom.current().nextInt(CHARS.length()));
- }
-
- return prefix + new String(buffer);
- }
-
- /** find the index where weightsUDF[index] < weight && weightsUDF[index+1] >= weight */
- private static int binarySearchIndex(long[] weightsUDF, long target) {
- Preconditions.checkArgument(
- target < weightsUDF[weightsUDF.length - 1],
- "weight is out of range: total weight = %s, search target = %s",
- weightsUDF[weightsUDF.length - 1],
- target);
- int start = 0;
- int end = weightsUDF.length - 1;
- while (start < end) {
- int mid = (start + end) / 2;
- if (weightsUDF[mid] < target && weightsUDF[mid + 1] >= target) {
- return mid;
- }
-
- if (weightsUDF[mid] >= target) {
- end = mid - 1;
- } else if (weightsUDF[mid + 1] < target) {
- start = mid + 1;
- }
- }
- return start;
- }
-
- /** Key is the id string and value is the weight in long value. */
- private static NavigableMap<Integer, Long> longTailDistribution(
- long startingWeight,
- int longTailStartingIndex,
- int longTailLength,
- long longTailBaseWeight,
- double weightRandomJitterPercentage) {
-
- NavigableMap<Integer, Long> weights = Maps.newTreeMap();
-
- // first part just decays the weight by half
- long currentWeight = startingWeight;
- for (int index = 0; index < longTailStartingIndex; ++index) {
- double jitter = ThreadLocalRandom.current().nextDouble(weightRandomJitterPercentage / 100);
- long weight = (long) (currentWeight * (1.0 + jitter));
- weight = weight > 0 ? weight : 1;
- weights.put(index, weight);
- if (currentWeight > longTailBaseWeight) {
- currentWeight = currentWeight / 2;
- }
- }
-
- // long tail part
- for (int index = longTailStartingIndex;
- index < longTailStartingIndex + longTailLength;
- ++index) {
- long longTailWeight =
- (long)
- (longTailBaseWeight
- * ThreadLocalRandom.current().nextDouble(weightRandomJitterPercentage));
- longTailWeight = longTailWeight > 0 ? longTailWeight : 1;
- weights.put(index, longTailWeight);
- }
-
- return weights;
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java
deleted file mode 100644
index 18473bf4f1..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/CatalogLoader.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.io.Serializable;
-import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.CatalogProperties;
-import org.apache.iceberg.CatalogUtil;
-import org.apache.iceberg.catalog.Catalog;
-import org.apache.iceberg.hadoop.HadoopCatalog;
-import org.apache.iceberg.hadoop.SerializableConfiguration;
-import org.apache.iceberg.hive.HiveCatalog;
-import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.rest.RESTCatalog;
-
-/** Serializable loader to load an Iceberg {@link Catalog}. */
-public interface CatalogLoader extends Serializable, Cloneable {
-
- /**
- * Create a new catalog with the provided properties. NOTICE: for flink, we may initialize the
- * {@link CatalogLoader} at flink sql client side or job manager side, and then serialize this
- * catalog loader to task manager, finally deserialize it and create a new catalog at task manager
- * side.
- *
- * @return a newly created {@link Catalog}
- */
- Catalog loadCatalog();
-
- /** Clone a CatalogLoader. */
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- CatalogLoader clone();
-
- static CatalogLoader hadoop(
- String name, Configuration hadoopConf, Map<String, String> properties) {
- return new HadoopCatalogLoader(name, hadoopConf, properties);
- }
-
- static CatalogLoader hive(String name, Configuration hadoopConf, Map<String, String> properties) {
- return new HiveCatalogLoader(name, hadoopConf, properties);
- }
-
- static CatalogLoader rest(String name, Configuration hadoopConf, Map<String, String> properties) {
- return new RESTCatalogLoader(name, hadoopConf, properties);
- }
-
- static CatalogLoader custom(
- String name, Map<String, String> properties, Configuration hadoopConf, String impl) {
- return new CustomCatalogLoader(name, properties, hadoopConf, impl);
- }
-
- class HadoopCatalogLoader implements CatalogLoader {
- private final String catalogName;
- private final SerializableConfiguration hadoopConf;
- private final String warehouseLocation;
- private final Map<String, String> properties;
-
- private HadoopCatalogLoader(
- String catalogName, Configuration conf, Map<String, String> properties) {
- this.catalogName = catalogName;
- this.hadoopConf = new SerializableConfiguration(conf);
- this.warehouseLocation = properties.get(CatalogProperties.WAREHOUSE_LOCATION);
- this.properties = Maps.newHashMap(properties);
- }
-
- @Override
- public Catalog loadCatalog() {
- return CatalogUtil.loadCatalog(
- HadoopCatalog.class.getName(), catalogName, properties, hadoopConf.get());
- }
-
- @Override
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- public CatalogLoader clone() {
- return new HadoopCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("catalogName", catalogName)
- .add("warehouseLocation", warehouseLocation)
- .toString();
- }
- }
-
- class HiveCatalogLoader implements CatalogLoader {
- private final String catalogName;
- private final SerializableConfiguration hadoopConf;
- private final String uri;
- private final String warehouse;
- private final int clientPoolSize;
- private final Map<String, String> properties;
-
- private HiveCatalogLoader(
- String catalogName, Configuration conf, Map<String, String> properties) {
- this.catalogName = catalogName;
- this.hadoopConf = new SerializableConfiguration(conf);
- this.uri = properties.get(CatalogProperties.URI);
- this.warehouse = properties.get(CatalogProperties.WAREHOUSE_LOCATION);
- this.clientPoolSize =
- properties.containsKey(CatalogProperties.CLIENT_POOL_SIZE)
- ? Integer.parseInt(properties.get(CatalogProperties.CLIENT_POOL_SIZE))
- : CatalogProperties.CLIENT_POOL_SIZE_DEFAULT;
- this.properties = Maps.newHashMap(properties);
- }
-
- @Override
- public Catalog loadCatalog() {
- return CatalogUtil.loadCatalog(
- HiveCatalog.class.getName(), catalogName, properties, hadoopConf.get());
- }
-
- @Override
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- public CatalogLoader clone() {
- return new HiveCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("catalogName", catalogName)
- .add("uri", uri)
- .add("warehouse", warehouse)
- .add("clientPoolSize", clientPoolSize)
- .toString();
- }
- }
-
- class RESTCatalogLoader implements CatalogLoader {
- private final String catalogName;
- private final SerializableConfiguration hadoopConf;
- private final Map<String, String> properties;
-
- private RESTCatalogLoader(
- String catalogName, Configuration conf, Map<String, String> properties) {
- this.catalogName = catalogName;
- this.hadoopConf = new SerializableConfiguration(conf);
- this.properties = Maps.newHashMap(properties);
- }
-
- @Override
- public Catalog loadCatalog() {
- return CatalogUtil.loadCatalog(
- RESTCatalog.class.getName(), catalogName, properties, hadoopConf.get());
- }
-
- @Override
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- public CatalogLoader clone() {
- return new RESTCatalogLoader(catalogName, new Configuration(hadoopConf.get()), properties);
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("catalogName", catalogName)
- .add("properties", properties)
- .toString();
- }
- }
-
- class CustomCatalogLoader implements CatalogLoader {
-
- private final SerializableConfiguration hadoopConf;
- private final Map<String, String> properties;
- private final String name;
- private final String impl;
-
- private CustomCatalogLoader(
- String name, Map<String, String> properties, Configuration conf, String impl) {
- this.hadoopConf = new SerializableConfiguration(conf);
- this.properties = Maps.newHashMap(properties); // wrap into a hashmap for serialization
- this.name = name;
- this.impl =
- Preconditions.checkNotNull(
- impl, "Cannot initialize custom Catalog, impl class name is null");
- }
-
- @Override
- public Catalog loadCatalog() {
- return CatalogUtil.loadCatalog(impl, name, properties, hadoopConf.get());
- }
-
- @Override
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- public CatalogLoader clone() {
- return new CustomCatalogLoader(name, properties, new Configuration(hadoopConf.get()), impl);
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this).add("name", name).add("impl", impl).toString();
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
deleted file mode 100644
index 825816fdf4..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
+++ /dev/null
@@ -1,806 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.catalog.AbstractCatalog;
-import org.apache.flink.table.catalog.CatalogBaseTable;
-import org.apache.flink.table.catalog.CatalogDatabase;
-import org.apache.flink.table.catalog.CatalogDatabaseImpl;
-import org.apache.flink.table.catalog.CatalogFunction;
-import org.apache.flink.table.catalog.CatalogPartition;
-import org.apache.flink.table.catalog.CatalogPartitionSpec;
-import org.apache.flink.table.catalog.CatalogTable;
-import org.apache.flink.table.catalog.CatalogTableImpl;
-import org.apache.flink.table.catalog.ObjectPath;
-import org.apache.flink.table.catalog.exceptions.CatalogException;
-import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
-import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
-import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
-import org.apache.flink.table.catalog.exceptions.FunctionNotExistException;
-import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
-import org.apache.flink.table.catalog.exceptions.TableNotExistException;
-import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException;
-import org.apache.flink.table.catalog.stats.CatalogColumnStatistics;
-import org.apache.flink.table.catalog.stats.CatalogTableStatistics;
-import org.apache.flink.table.expressions.Expression;
-import org.apache.flink.table.factories.Factory;
-import org.apache.flink.util.StringUtils;
-import org.apache.iceberg.CachingCatalog;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.FileScanTask;
-import org.apache.iceberg.MetadataTableType;
-import org.apache.iceberg.PartitionField;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.StructLike;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.Transaction;
-import org.apache.iceberg.UpdateProperties;
-import org.apache.iceberg.catalog.Catalog;
-import org.apache.iceberg.catalog.Namespace;
-import org.apache.iceberg.catalog.SupportsNamespaces;
-import org.apache.iceberg.catalog.TableIdentifier;
-import org.apache.iceberg.exceptions.AlreadyExistsException;
-import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
-import org.apache.iceberg.exceptions.NoSuchNamespaceException;
-import org.apache.iceberg.flink.util.FlinkCompatibilityUtil;
-import org.apache.iceberg.io.CloseableIterable;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.base.Splitter;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-
-/**
- * A Flink Catalog implementation that wraps an Iceberg {@link Catalog}.
- *
- * <p>The mapping between Flink database and Iceberg namespace: Supplying a base namespace for a
- * given catalog, so if you have a catalog that supports a 2-level namespace, you would supply the
- * first level in the catalog configuration and the second level would be exposed as Flink
- * databases.
- *
- * <p>The Iceberg table manages its partitions by itself. The partition of the Iceberg table is
- * independent of the partition of Flink.
- */
-public class FlinkCatalog extends AbstractCatalog {
-
- private final CatalogLoader catalogLoader;
- private final Catalog icebergCatalog;
- private final Namespace baseNamespace;
- private final SupportsNamespaces asNamespaceCatalog;
- private final Closeable closeable;
- private final boolean cacheEnabled;
-
- public FlinkCatalog(
- String catalogName,
- String defaultDatabase,
- Namespace baseNamespace,
- CatalogLoader catalogLoader,
- boolean cacheEnabled,
- long cacheExpirationIntervalMs) {
- super(catalogName, defaultDatabase);
- this.catalogLoader = catalogLoader;
- this.baseNamespace = baseNamespace;
- this.cacheEnabled = cacheEnabled;
-
- Catalog originalCatalog = catalogLoader.loadCatalog();
- icebergCatalog =
- cacheEnabled
- ? CachingCatalog.wrap(originalCatalog, cacheExpirationIntervalMs)
- : originalCatalog;
- asNamespaceCatalog =
- originalCatalog instanceof SupportsNamespaces ? (SupportsNamespaces) originalCatalog : null;
- closeable = originalCatalog instanceof Closeable ? (Closeable) originalCatalog : null;
-
- FlinkEnvironmentContext.init();
- }
-
- @Override
- public void open() throws CatalogException {}
-
- @Override
- public void close() throws CatalogException {
- if (closeable != null) {
- try {
- closeable.close();
- } catch (IOException e) {
- throw new CatalogException(e);
- }
- }
- }
-
- public Catalog catalog() {
- return icebergCatalog;
- }
-
- /** Append a new level to the base namespace */
- private static Namespace appendLevel(Namespace baseNamespace, String newLevel) {
- String[] namespace = new String[baseNamespace.levels().length + 1];
- System.arraycopy(baseNamespace.levels(), 0, namespace, 0, baseNamespace.levels().length);
- namespace[baseNamespace.levels().length] = newLevel;
- return Namespace.of(namespace);
- }
-
- TableIdentifier toIdentifier(ObjectPath path) {
- String objectName = path.getObjectName();
- List<String> tableName = Splitter.on('$').splitToList(objectName);
-
- if (tableName.size() == 1) {
- return TableIdentifier.of(
- appendLevel(baseNamespace, path.getDatabaseName()), path.getObjectName());
- } else if (tableName.size() == 2 && MetadataTableType.from(tableName.get(1)) != null) {
- return TableIdentifier.of(
- appendLevel(appendLevel(baseNamespace, path.getDatabaseName()), tableName.get(0)),
- tableName.get(1));
- } else {
- throw new IllegalArgumentException("Illegal table name:" + objectName);
- }
- }
-
- @Override
- public List<String> listDatabases() throws CatalogException {
- if (asNamespaceCatalog == null) {
- return Collections.singletonList(getDefaultDatabase());
- }
-
- return asNamespaceCatalog.listNamespaces(baseNamespace).stream()
- .map(n -> n.level(n.levels().length - 1))
- .collect(Collectors.toList());
- }
-
- @Override
- public CatalogDatabase getDatabase(String databaseName)
- throws DatabaseNotExistException, CatalogException {
- if (asNamespaceCatalog == null) {
- if (!getDefaultDatabase().equals(databaseName)) {
- throw new DatabaseNotExistException(getName(), databaseName);
- } else {
- return new CatalogDatabaseImpl(Maps.newHashMap(), "");
- }
- } else {
- try {
- Map<String, String> metadata =
- Maps.newHashMap(
- asNamespaceCatalog.loadNamespaceMetadata(appendLevel(baseNamespace, databaseName)));
- String comment = metadata.remove("comment");
- return new CatalogDatabaseImpl(metadata, comment);
- } catch (NoSuchNamespaceException e) {
- throw new DatabaseNotExistException(getName(), databaseName, e);
- }
- }
- }
-
- @Override
- public boolean databaseExists(String databaseName) throws CatalogException {
- try {
- getDatabase(databaseName);
- return true;
- } catch (DatabaseNotExistException ignore) {
- return false;
- }
- }
-
- @Override
- public void createDatabase(String name, CatalogDatabase database, boolean ignoreIfExists)
- throws DatabaseAlreadyExistException, CatalogException {
- createDatabase(
- name, mergeComment(database.getProperties(), database.getComment()), ignoreIfExists);
- }
-
- private void createDatabase(
- String databaseName, Map<String, String> metadata, boolean ignoreIfExists)
- throws DatabaseAlreadyExistException, CatalogException {
- if (asNamespaceCatalog != null) {
- try {
- asNamespaceCatalog.createNamespace(appendLevel(baseNamespace, databaseName), metadata);
- } catch (AlreadyExistsException e) {
- if (!ignoreIfExists) {
- throw new DatabaseAlreadyExistException(getName(), databaseName, e);
- }
- }
- } else {
- throw new UnsupportedOperationException(
- "Namespaces are not supported by catalog: " + getName());
- }
- }
-
- private Map<String, String> mergeComment(Map<String, String> metadata, String comment) {
- Map<String, String> ret = Maps.newHashMap(metadata);
- if (metadata.containsKey("comment")) {
- throw new CatalogException("Database properties should not contain key: 'comment'.");
- }
-
- if (!StringUtils.isNullOrWhitespaceOnly(comment)) {
- ret.put("comment", comment);
- }
- return ret;
- }
-
- @Override
- public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade)
- throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException {
- if (asNamespaceCatalog != null) {
- try {
- boolean success = asNamespaceCatalog.dropNamespace(appendLevel(baseNamespace, name));
- if (!success && !ignoreIfNotExists) {
- throw new DatabaseNotExistException(getName(), name);
- }
- } catch (NoSuchNamespaceException e) {
- if (!ignoreIfNotExists) {
- throw new DatabaseNotExistException(getName(), name, e);
- }
- } catch (NamespaceNotEmptyException e) {
- throw new DatabaseNotEmptyException(getName(), name, e);
- }
- } else {
- if (!ignoreIfNotExists) {
- throw new DatabaseNotExistException(getName(), name);
- }
- }
- }
-
- @Override
- public void alterDatabase(String name, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
- throws DatabaseNotExistException, CatalogException {
- if (asNamespaceCatalog != null) {
- Namespace namespace = appendLevel(baseNamespace, name);
- Map<String, String> updates = Maps.newHashMap();
- Set<String> removals = Sets.newHashSet();
-
- try {
- Map<String, String> oldProperties = asNamespaceCatalog.loadNamespaceMetadata(namespace);
- Map<String, String> newProperties =
- mergeComment(newDatabase.getProperties(), newDatabase.getComment());
-
- for (String key : oldProperties.keySet()) {
- if (!newProperties.containsKey(key)) {
- removals.add(key);
- }
- }
-
- for (Map.Entry<String, String> entry : newProperties.entrySet()) {
- if (!entry.getValue().equals(oldProperties.get(entry.getKey()))) {
- updates.put(entry.getKey(), entry.getValue());
- }
- }
-
- if (!updates.isEmpty()) {
- asNamespaceCatalog.setProperties(namespace, updates);
- }
-
- if (!removals.isEmpty()) {
- asNamespaceCatalog.removeProperties(namespace, removals);
- }
-
- } catch (NoSuchNamespaceException e) {
- if (!ignoreIfNotExists) {
- throw new DatabaseNotExistException(getName(), name, e);
- }
- }
- } else {
- if (getDefaultDatabase().equals(name)) {
- throw new CatalogException(
- "Can not alter the default database when the iceberg catalog doesn't support namespaces.");
- }
- if (!ignoreIfNotExists) {
- throw new DatabaseNotExistException(getName(), name);
- }
- }
- }
-
- @Override
- public List<String> listTables(String databaseName)
- throws DatabaseNotExistException, CatalogException {
- try {
- return icebergCatalog.listTables(appendLevel(baseNamespace, databaseName)).stream()
- .map(TableIdentifier::name)
- .collect(Collectors.toList());
- } catch (NoSuchNamespaceException e) {
- throw new DatabaseNotExistException(getName(), databaseName, e);
- }
- }
-
- @Override
- public CatalogTable getTable(ObjectPath tablePath)
- throws TableNotExistException, CatalogException {
- Table table = loadIcebergTable(tablePath);
- return toCatalogTable(table);
- }
-
- private Table loadIcebergTable(ObjectPath tablePath) throws TableNotExistException {
- try {
- Table table = icebergCatalog.loadTable(toIdentifier(tablePath));
- if (cacheEnabled) {
- table.refresh();
- }
-
- return table;
- } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
- throw new TableNotExistException(getName(), tablePath, e);
- }
- }
-
- @Override
- public boolean tableExists(ObjectPath tablePath) throws CatalogException {
- return icebergCatalog.tableExists(toIdentifier(tablePath));
- }
-
- @Override
- public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
- throws TableNotExistException, CatalogException {
- try {
- icebergCatalog.dropTable(toIdentifier(tablePath));
- } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
- if (!ignoreIfNotExists) {
- throw new TableNotExistException(getName(), tablePath, e);
- }
- }
- }
-
- @Override
- public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
- throws TableNotExistException, TableAlreadyExistException, CatalogException {
- try {
- icebergCatalog.renameTable(
- toIdentifier(tablePath),
- toIdentifier(new ObjectPath(tablePath.getDatabaseName(), newTableName)));
- } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
- if (!ignoreIfNotExists) {
- throw new TableNotExistException(getName(), tablePath, e);
- }
- } catch (AlreadyExistsException e) {
- throw new TableAlreadyExistException(getName(), tablePath, e);
- }
- }
-
- @Override
- public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
- throws CatalogException, TableAlreadyExistException {
- if (Objects.equals(
- table.getOptions().get("connector"), FlinkDynamicTableFactory.FACTORY_IDENTIFIER)) {
- throw new IllegalArgumentException(
- "Cannot create the table with 'connector'='iceberg' table property in "
- + "an iceberg catalog, Please create table with 'connector'='iceberg' property in a non-iceberg catalog or "
- + "create table without 'connector'='iceberg' related properties in an iceberg table.");
- }
-
- createIcebergTable(tablePath, table, ignoreIfExists);
- }
-
- void createIcebergTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
- throws CatalogException, TableAlreadyExistException {
- validateFlinkTable(table);
-
- Schema icebergSchema = FlinkSchemaUtil.convert(table.getSchema());
- PartitionSpec spec = toPartitionSpec(((CatalogTable) table).getPartitionKeys(), icebergSchema);
-
- ImmutableMap.Builder<String, String> properties = ImmutableMap.builder();
- String location = null;
- for (Map.Entry<String, String> entry : table.getOptions().entrySet()) {
- if ("location".equalsIgnoreCase(entry.getKey())) {
- location = entry.getValue();
- } else {
- properties.put(entry.getKey(), entry.getValue());
- }
- }
-
- try {
- icebergCatalog.createTable(
- toIdentifier(tablePath), icebergSchema, spec, location, properties.build());
- } catch (AlreadyExistsException e) {
- if (!ignoreIfExists) {
- throw new TableAlreadyExistException(getName(), tablePath, e);
- }
- }
- }
-
- private static void validateTableSchemaAndPartition(CatalogTable ct1, CatalogTable ct2) {
- TableSchema ts1 = ct1.getSchema();
- TableSchema ts2 = ct2.getSchema();
- boolean equalsPrimary = false;
-
- if (ts1.getPrimaryKey().isPresent() && ts2.getPrimaryKey().isPresent()) {
- equalsPrimary =
- Objects.equals(ts1.getPrimaryKey().get().getType(), ts2.getPrimaryKey().get().getType())
- && Objects.equals(
- ts1.getPrimaryKey().get().getColumns(), ts2.getPrimaryKey().get().getColumns());
- } else if (!ts1.getPrimaryKey().isPresent() && !ts2.getPrimaryKey().isPresent()) {
- equalsPrimary = true;
- }
-
- if (!(Objects.equals(ts1.getTableColumns(), ts2.getTableColumns())
- && Objects.equals(ts1.getWatermarkSpecs(), ts2.getWatermarkSpecs())
- && equalsPrimary)) {
- throw new UnsupportedOperationException("Altering schema is not supported yet.");
- }
-
- if (!ct1.getPartitionKeys().equals(ct2.getPartitionKeys())) {
- throw new UnsupportedOperationException("Altering partition keys is not supported yet.");
- }
- }
-
- @Override
- public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
- throws CatalogException, TableNotExistException {
- validateFlinkTable(newTable);
-
- Table icebergTable;
- try {
- icebergTable = loadIcebergTable(tablePath);
- } catch (TableNotExistException e) {
- if (!ignoreIfNotExists) {
- throw e;
- } else {
- return;
- }
- }
-
- CatalogTable table = toCatalogTable(icebergTable);
-
- // Currently, Flink SQL only support altering table properties.
-
- // For current Flink Catalog API, support for adding/removing/renaming columns cannot be done by
- // comparing
- // CatalogTable instances, unless the Flink schema contains Iceberg column IDs.
- validateTableSchemaAndPartition(table, (CatalogTable) newTable);
-
- Map<String, String> oldProperties = table.getOptions();
- Map<String, String> setProperties = Maps.newHashMap();
-
- String setLocation = null;
- String setSnapshotId = null;
- String pickSnapshotId = null;
-
- for (Map.Entry<String, String> entry : newTable.getOptions().entrySet()) {
- String key = entry.getKey();
- String value = entry.getValue();
-
- if (Objects.equals(value, oldProperties.get(key))) {
- continue;
- }
-
- if ("location".equalsIgnoreCase(key)) {
- setLocation = value;
- } else if ("current-snapshot-id".equalsIgnoreCase(key)) {
- setSnapshotId = value;
- } else if ("cherry-pick-snapshot-id".equalsIgnoreCase(key)) {
- pickSnapshotId = value;
- } else {
- setProperties.put(key, value);
- }
- }
-
- oldProperties
- .keySet()
- .forEach(
- k -> {
- if (!newTable.getOptions().containsKey(k)) {
- setProperties.put(k, null);
- }
- });
-
- commitChanges(icebergTable, setLocation, setSnapshotId, pickSnapshotId, setProperties);
- }
-
- private static void validateFlinkTable(CatalogBaseTable table) {
- Preconditions.checkArgument(
- table instanceof CatalogTable, "The Table should be a CatalogTable.");
-
- TableSchema schema = table.getSchema();
- schema
- .getTableColumns()
- .forEach(
- column -> {
- if (!FlinkCompatibilityUtil.isPhysicalColumn(column)) {
- throw new UnsupportedOperationException(
- "Creating table with computed columns is not supported yet.");
- }
- });
-
- if (!schema.getWatermarkSpecs().isEmpty()) {
- throw new UnsupportedOperationException(
- "Creating table with watermark specs is not supported yet.");
- }
- }
-
- private static PartitionSpec toPartitionSpec(List<String> partitionKeys, Schema icebergSchema) {
- PartitionSpec.Builder builder = PartitionSpec.builderFor(icebergSchema);
- partitionKeys.forEach(builder::identity);
- return builder.build();
- }
-
- private static List<String> toPartitionKeys(PartitionSpec spec, Schema icebergSchema) {
- ImmutableList.Builder<String> partitionKeysBuilder = ImmutableList.builder();
- for (PartitionField field : spec.fields()) {
- if (field.transform().isIdentity()) {
- partitionKeysBuilder.add(icebergSchema.findColumnName(field.sourceId()));
- } else {
- // Not created by Flink SQL.
- // For compatibility with iceberg tables, return empty.
- // TODO modify this after Flink support partition transform.
- return Collections.emptyList();
- }
- }
- return partitionKeysBuilder.build();
- }
-
- private static void commitChanges(
- Table table,
- String setLocation,
- String setSnapshotId,
- String pickSnapshotId,
- Map<String, String> setProperties) {
- // don't allow setting the snapshot and picking a commit at the same time because order is
- // ambiguous and choosing
- // one order leads to different results
- Preconditions.checkArgument(
- setSnapshotId == null || pickSnapshotId == null,
- "Cannot set the current snapshot ID and cherry-pick snapshot changes");
-
- if (setSnapshotId != null) {
- long newSnapshotId = Long.parseLong(setSnapshotId);
- table.manageSnapshots().setCurrentSnapshot(newSnapshotId).commit();
- }
-
- // if updating the table snapshot, perform that update first in case it fails
- if (pickSnapshotId != null) {
- long newSnapshotId = Long.parseLong(pickSnapshotId);
- table.manageSnapshots().cherrypick(newSnapshotId).commit();
- }
-
- Transaction transaction = table.newTransaction();
-
- if (setLocation != null) {
- transaction.updateLocation().setLocation(setLocation).commit();
- }
-
- if (!setProperties.isEmpty()) {
- UpdateProperties updateProperties = transaction.updateProperties();
- setProperties.forEach(
- (k, v) -> {
- if (v == null) {
- updateProperties.remove(k);
- } else {
- updateProperties.set(k, v);
- }
- });
- updateProperties.commit();
- }
-
- transaction.commitTransaction();
- }
-
- static CatalogTable toCatalogTable(Table table) {
- TableSchema schema = FlinkSchemaUtil.toSchema(table.schema());
- List<String> partitionKeys = toPartitionKeys(table.spec(), table.schema());
-
- // NOTE: We can not create a IcebergCatalogTable extends CatalogTable, because Flink optimizer
- // may use
- // CatalogTableImpl to copy a new catalog table.
- // Let's re-loading table from Iceberg catalog when creating source/sink operators.
- // Iceberg does not have Table comment, so pass a null (Default comment value in Flink).
- return new CatalogTableImpl(schema, partitionKeys, table.properties(), null);
- }
-
- @Override
- public Optional<Factory> getFactory() {
- return Optional.of(new FlinkDynamicTableFactory(this));
- }
-
- CatalogLoader getCatalogLoader() {
- return catalogLoader;
- }
-
- // ------------------------------ Unsupported methods
- // ---------------------------------------------
-
- @Override
- public List<String> listViews(String databaseName) throws CatalogException {
- return Collections.emptyList();
- }
-
- @Override
- public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void createPartition(
- ObjectPath tablePath,
- CatalogPartitionSpec partitionSpec,
- CatalogPartition partition,
- boolean ignoreIfExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void dropPartition(
- ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void alterPartition(
- ObjectPath tablePath,
- CatalogPartitionSpec partitionSpec,
- CatalogPartition newPartition,
- boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<String> listFunctions(String dbName) throws CatalogException {
- return Collections.emptyList();
- }
-
- @Override
- public CatalogFunction getFunction(ObjectPath functionPath)
- throws FunctionNotExistException, CatalogException {
- throw new FunctionNotExistException(getName(), functionPath);
- }
-
- @Override
- public boolean functionExists(ObjectPath functionPath) throws CatalogException {
- return false;
- }
-
- @Override
- public void createFunction(
- ObjectPath functionPath, CatalogFunction function, boolean ignoreIfExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void alterFunction(
- ObjectPath functionPath, CatalogFunction newFunction, boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void alterTableStatistics(
- ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void alterTableColumnStatistics(
- ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void alterPartitionStatistics(
- ObjectPath tablePath,
- CatalogPartitionSpec partitionSpec,
- CatalogTableStatistics partitionStatistics,
- boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void alterPartitionColumnStatistics(
- ObjectPath tablePath,
- CatalogPartitionSpec partitionSpec,
- CatalogColumnStatistics columnStatistics,
- boolean ignoreIfNotExists)
- throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
- throws TableNotExistException, TableNotPartitionedException, CatalogException {
- Table table = loadIcebergTable(tablePath);
-
- if (table.spec().isUnpartitioned()) {
- throw new TableNotPartitionedException(icebergCatalog.name(), tablePath);
- }
-
- Set<CatalogPartitionSpec> set = Sets.newHashSet();
- try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
- for (DataFile dataFile : CloseableIterable.transform(tasks, FileScanTask::file)) {
- Map<String, String> map = Maps.newHashMap();
- StructLike structLike = dataFile.partition();
- PartitionSpec spec = table.specs().get(dataFile.specId());
- for (int i = 0; i < structLike.size(); i++) {
- map.put(spec.fields().get(i).name(), String.valueOf(structLike.get(i, Object.class)));
- }
- set.add(new CatalogPartitionSpec(map));
- }
- } catch (IOException e) {
- throw new CatalogException(
- String.format("Failed to list partitions of table %s", tablePath), e);
- }
-
- return Lists.newArrayList(set);
- }
-
- @Override
- public List<CatalogPartitionSpec> listPartitions(
- ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<CatalogPartitionSpec> listPartitionsByFilter(
- ObjectPath tablePath, List<Expression> filters) throws CatalogException {
- throw new UnsupportedOperationException();
- }
-
- // After partition pruning and filter push down, the statistics have become very inaccurate, so
- // the statistics from
- // here are of little significance.
- // Flink will support something like SupportsReportStatistics in future.
-
- @Override
- public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws CatalogException {
- return CatalogTableStatistics.UNKNOWN;
- }
-
- @Override
- public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath)
- throws CatalogException {
- return CatalogColumnStatistics.UNKNOWN;
- }
-
- @Override
- public CatalogTableStatistics getPartitionStatistics(
- ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
- return CatalogTableStatistics.UNKNOWN;
- }
-
- @Override
- public CatalogColumnStatistics getPartitionColumnStatistics(
- ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
- return CatalogColumnStatistics.UNKNOWN;
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java
deleted file mode 100644
index 1453753849..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.runtime.util.HadoopUtils;
-import org.apache.flink.table.catalog.Catalog;
-import org.apache.flink.table.factories.CatalogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.iceberg.CatalogProperties;
-import org.apache.iceberg.catalog.Namespace;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.base.Strings;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.util.PropertyUtil;
-
-/**
- * A Flink Catalog factory implementation that creates {@link FlinkCatalog}.
- *
- * <p>This supports the following catalog configuration options:
- *
- * <ul>
- * <li><code>type</code> - Flink catalog factory key, should be "iceberg"
- * <li><code>catalog-type</code> - iceberg catalog type, "hive", "hadoop" or "rest"
- * <li><code>uri</code> - the Hive Metastore URI (Hive catalog only)
- * <li><code>clients</code> - the Hive Client Pool Size (Hive catalog only)
- * <li><code>warehouse</code> - the warehouse path (Hadoop catalog only)
- * <li><code>default-database</code> - a database name to use as the default
- * <li><code>base-namespace</code> - a base namespace as the prefix for all databases (Hadoop
- * catalog only)
- * <li><code>cache-enabled</code> - whether to enable catalog cache
- * </ul>
- *
- * <p>To use a custom catalog that is not a Hive or Hadoop catalog, extend this class and override
- * {@link #createCatalogLoader(String, Map, Configuration)}.
- */
-public class FlinkCatalogFactory implements CatalogFactory {
-
- // Can not just use "type", it conflicts with CATALOG_TYPE.
- public static final String ICEBERG_CATALOG_TYPE = "catalog-type";
- public static final String ICEBERG_CATALOG_TYPE_HADOOP = "hadoop";
- public static final String ICEBERG_CATALOG_TYPE_HIVE = "hive";
- public static final String ICEBERG_CATALOG_TYPE_REST = "rest";
-
- public static final String HIVE_CONF_DIR = "hive-conf-dir";
- public static final String HADOOP_CONF_DIR = "hadoop-conf-dir";
- public static final String DEFAULT_DATABASE = "default-database";
- public static final String DEFAULT_DATABASE_NAME = "default";
- public static final String BASE_NAMESPACE = "base-namespace";
-
- public static final String TYPE = "type";
- public static final String PROPERTY_VERSION = "property-version";
-
- /**
- * Create an Iceberg {@link org.apache.iceberg.catalog.Catalog} loader to be used by this Flink
- * catalog adapter.
- *
- * @param name Flink's catalog name
- * @param properties Flink's catalog properties
- * @param hadoopConf Hadoop configuration for catalog
- * @return an Iceberg catalog loader
- */
- static CatalogLoader createCatalogLoader(
- String name, Map<String, String> properties, Configuration hadoopConf) {
- String catalogImpl = properties.get(CatalogProperties.CATALOG_IMPL);
- if (catalogImpl != null) {
- String catalogType = properties.get(ICEBERG_CATALOG_TYPE);
- Preconditions.checkArgument(
- catalogType == null,
- "Cannot create catalog %s, both catalog-type and catalog-impl are set: catalog-type=%s, catalog-impl=%s",
- name,
- catalogType,
- catalogImpl);
- return CatalogLoader.custom(name, properties, hadoopConf, catalogImpl);
- }
-
- String catalogType = properties.getOrDefault(ICEBERG_CATALOG_TYPE, ICEBERG_CATALOG_TYPE_HIVE);
- switch (catalogType.toLowerCase(Locale.ENGLISH)) {
- case ICEBERG_CATALOG_TYPE_HIVE:
- // The values of properties 'uri', 'warehouse', 'hive-conf-dir' are allowed to be null, in
- // that case it will
- // fallback to parse those values from hadoop configuration which is loaded from classpath.
- String hiveConfDir = properties.get(HIVE_CONF_DIR);
- String hadoopConfDir = properties.get(HADOOP_CONF_DIR);
- Configuration newHadoopConf = mergeHiveConf(hadoopConf, hiveConfDir, hadoopConfDir);
- return CatalogLoader.hive(name, newHadoopConf, properties);
-
- case ICEBERG_CATALOG_TYPE_HADOOP:
- return CatalogLoader.hadoop(name, hadoopConf, properties);
-
- case ICEBERG_CATALOG_TYPE_REST:
- return CatalogLoader.rest(name, hadoopConf, properties);
-
- default:
- throw new UnsupportedOperationException(
- "Unknown catalog-type: " + catalogType + " (Must be 'hive', 'hadoop' or 'rest')");
- }
- }
-
- @Override
- public Map<String, String> requiredContext() {
- Map<String, String> context = Maps.newHashMap();
- context.put(TYPE, "iceberg");
- context.put(PROPERTY_VERSION, "1");
- return context;
- }
-
- @Override
- public List<String> supportedProperties() {
- return ImmutableList.of("*");
- }
-
- @Override
- public Catalog createCatalog(String name, Map<String, String> properties) {
- return createCatalog(name, properties, clusterHadoopConf());
- }
-
- protected Catalog createCatalog(
- String name, Map<String, String> properties, Configuration hadoopConf) {
- CatalogLoader catalogLoader = createCatalogLoader(name, properties, hadoopConf);
- String defaultDatabase = properties.getOrDefault(DEFAULT_DATABASE, DEFAULT_DATABASE_NAME);
-
- Namespace baseNamespace = Namespace.empty();
- if (properties.containsKey(BASE_NAMESPACE)) {
- baseNamespace = Namespace.of(properties.get(BASE_NAMESPACE).split("\\."));
- }
-
- boolean cacheEnabled =
- PropertyUtil.propertyAsBoolean(
- properties, CatalogProperties.CACHE_ENABLED, CatalogProperties.CACHE_ENABLED_DEFAULT);
-
- long cacheExpirationIntervalMs =
- PropertyUtil.propertyAsLong(
- properties,
- CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS,
- CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS_OFF);
- Preconditions.checkArgument(
- cacheExpirationIntervalMs != 0,
- "%s is not allowed to be 0.",
- CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS);
-
- return new FlinkCatalog(
- name,
- defaultDatabase,
- baseNamespace,
- catalogLoader,
- cacheEnabled,
- cacheExpirationIntervalMs);
- }
-
- private static Configuration mergeHiveConf(
- Configuration hadoopConf, String hiveConfDir, String hadoopConfDir) {
- Configuration newConf = new Configuration(hadoopConf);
- if (!Strings.isNullOrEmpty(hiveConfDir)) {
- Preconditions.checkState(
- Files.exists(Paths.get(hiveConfDir, "hive-site.xml")),
- "There should be a hive-site.xml file under the directory %s",
- hiveConfDir);
- newConf.addResource(new Path(hiveConfDir, "hive-site.xml"));
- } else {
- // If don't provide the hive-site.xml path explicitly, it will try to load resource from
- // classpath. If still
- // couldn't load the configuration file, then it will throw exception in HiveCatalog.
- URL configFile = CatalogLoader.class.getClassLoader().getResource("hive-site.xml");
- if (configFile != null) {
- newConf.addResource(configFile);
- }
- }
-
- if (!Strings.isNullOrEmpty(hadoopConfDir)) {
- Preconditions.checkState(
- Files.exists(Paths.get(hadoopConfDir, "hdfs-site.xml")),
- "Failed to load Hadoop configuration: missing %s",
- Paths.get(hadoopConfDir, "hdfs-site.xml"));
- newConf.addResource(new Path(hadoopConfDir, "hdfs-site.xml"));
- Preconditions.checkState(
- Files.exists(Paths.get(hadoopConfDir, "core-site.xml")),
- "Failed to load Hadoop configuration: missing %s",
- Paths.get(hadoopConfDir, "core-site.xml"));
- newConf.addResource(new Path(hadoopConfDir, "core-site.xml"));
- }
-
- return newConf;
- }
-
- public static Configuration clusterHadoopConf() {
- return HadoopUtils.getHadoopConfiguration(GlobalConfiguration.loadConfiguration());
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java
deleted file mode 100644
index 7167859e60..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.time.Duration;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.flink.util.TimeUtils;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-
-class FlinkConfParser {
-
- private final Map<String, String> tableProperties;
- private final Map<String, String> options;
- private final ReadableConfig readableConfig;
-
- FlinkConfParser(Table table, Map<String, String> options, ReadableConfig readableConfig) {
- this.tableProperties = table.properties();
- this.options = options;
- this.readableConfig = readableConfig;
- }
-
- public BooleanConfParser booleanConf() {
- return new BooleanConfParser();
- }
-
- public IntConfParser intConf() {
- return new IntConfParser();
- }
-
- public LongConfParser longConf() {
- return new LongConfParser();
- }
-
- public <E extends Enum<E>> EnumConfParser<E> enumConfParser(Class<E> enumClass) {
- return new EnumConfParser<>(enumClass);
- }
-
- public StringConfParser stringConf() {
- return new StringConfParser();
- }
-
- public DurationConfParser durationConf() {
- return new DurationConfParser();
- }
-
- class BooleanConfParser extends ConfParser<BooleanConfParser, Boolean> {
- private Boolean defaultValue;
-
- @Override
- protected BooleanConfParser self() {
- return this;
- }
-
- public BooleanConfParser defaultValue(boolean value) {
- this.defaultValue = value;
- return self();
- }
-
- public BooleanConfParser defaultValue(String value) {
- this.defaultValue = Boolean.parseBoolean(value);
- return self();
- }
-
- public boolean parse() {
- Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
- return parse(Boolean::parseBoolean, defaultValue);
- }
- }
-
- class IntConfParser extends ConfParser<IntConfParser, Integer> {
- private Integer defaultValue;
-
- @Override
- protected IntConfParser self() {
- return this;
- }
-
- public IntConfParser defaultValue(int value) {
- this.defaultValue = value;
- return self();
- }
-
- public int parse() {
- Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
- return parse(Integer::parseInt, defaultValue);
- }
-
- public Integer parseOptional() {
- return parse(Integer::parseInt, null);
- }
- }
-
- class LongConfParser extends ConfParser<LongConfParser, Long> {
- private Long defaultValue;
-
- @Override
- protected LongConfParser self() {
- return this;
- }
-
- public LongConfParser defaultValue(long value) {
- this.defaultValue = value;
- return self();
- }
-
- public long parse() {
- Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
- return parse(Long::parseLong, defaultValue);
- }
-
- public Long parseOptional() {
- return parse(Long::parseLong, null);
- }
- }
-
- class StringConfParser extends ConfParser<StringConfParser, String> {
- private String defaultValue;
-
- @Override
- protected StringConfParser self() {
- return this;
- }
-
- public StringConfParser defaultValue(String value) {
- this.defaultValue = value;
- return self();
- }
-
- public String parse() {
- Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
- return parse(Function.identity(), defaultValue);
- }
-
- public String parseOptional() {
- return parse(Function.identity(), null);
- }
- }
-
- class EnumConfParser<E extends Enum<E>> extends ConfParser<EnumConfParser<E>, E> {
- private E defaultValue;
- private final Class<E> enumClass;
-
- EnumConfParser(Class<E> enumClass) {
- this.enumClass = enumClass;
- }
-
- @Override
- protected EnumConfParser<E> self() {
- return this;
- }
-
- public EnumConfParser<E> defaultValue(E value) {
- this.defaultValue = value;
- return self();
- }
-
- public E parse() {
- Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
- return parse(s -> Enum.valueOf(enumClass, s), defaultValue);
- }
-
- public E parseOptional() {
- return parse(s -> Enum.valueOf(enumClass, s), null);
- }
- }
-
- class DurationConfParser extends ConfParser<DurationConfParser, Duration> {
- private Duration defaultValue;
-
- @Override
- protected DurationConfParser self() {
- return this;
- }
-
- public DurationConfParser defaultValue(Duration value) {
- this.defaultValue = value;
- return self();
- }
-
- public Duration parse() {
- Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
- return parse(TimeUtils::parseDuration, defaultValue);
- }
-
- public Duration parseOptional() {
- return parse(TimeUtils::parseDuration, null);
- }
- }
-
- abstract class ConfParser<ThisT, T> {
- private final List<String> optionNames = Lists.newArrayList();
- private String tablePropertyName;
- private ConfigOption<T> configOption;
-
- protected abstract ThisT self();
-
- public ThisT option(String name) {
- this.optionNames.add(name);
- return self();
- }
-
- public ThisT flinkConfig(ConfigOption<T> newConfigOption) {
- this.configOption = newConfigOption;
- return self();
- }
-
- public ThisT tableProperty(String name) {
- this.tablePropertyName = name;
- return self();
- }
-
- protected T parse(Function<String, T> conversion, T defaultValue) {
- if (!optionNames.isEmpty()) {
- for (String optionName : optionNames) {
- String optionValue = options.get(optionName);
- if (optionValue != null) {
- return conversion.apply(optionValue);
- }
- }
- }
-
- if (configOption != null) {
- T propertyValue = readableConfig.get(configOption);
- if (propertyValue != null) {
- return propertyValue;
- }
- }
-
- if (tablePropertyName != null) {
- String propertyValue = tableProperties.get(tablePropertyName);
- if (propertyValue != null) {
- return conversion.apply(propertyValue);
- }
- }
-
- return defaultValue;
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java
deleted file mode 100644
index 7c7afd24ed..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkConfigOptions.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.description.Description;
-import org.apache.flink.configuration.description.TextElement;
-import org.apache.flink.table.api.TableEnvironment;
-import org.apache.iceberg.flink.source.assigner.SplitAssignerType;
-import org.apache.iceberg.util.ThreadPools;
-
-/**
- * When constructing Flink Iceberg source via Java API, configs can be set in {@link Configuration}
- * passed to source builder. E.g.
- *
- * <pre>
- * configuration.setBoolean(FlinkConfigOptions.TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM, true);
- * FlinkSource.forRowData()
- * .flinkConf(configuration)
- * ...
- * </pre>
- *
- * <p>When using Flink SQL/table API, connector options can be set in Flink's {@link
- * TableEnvironment}.
- *
- * <pre>
- * TableEnvironment tEnv = createTableEnv();
- * tEnv.getConfig()
- * .getConfiguration()
- * .setBoolean(FlinkConfigOptions.TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM, true);
- * </pre>
- */
-public class FlinkConfigOptions {
-
- private FlinkConfigOptions() {}
-
- public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM =
- ConfigOptions.key("table.exec.iceberg.infer-source-parallelism")
- .booleanType()
- .defaultValue(true)
- .withDescription(
- "If is false, parallelism of source are set by config.\n"
- + "If is true, source parallelism is inferred according to splits number.\n");
-
- public static final ConfigOption<Integer> TABLE_EXEC_ICEBERG_INFER_SOURCE_PARALLELISM_MAX =
- ConfigOptions.key("table.exec.iceberg.infer-source-parallelism.max")
- .intType()
- .defaultValue(100)
- .withDescription("Sets max infer parallelism for source operator.");
-
- public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_EXPOSE_SPLIT_LOCALITY_INFO =
- ConfigOptions.key("table.exec.iceberg.expose-split-locality-info")
- .booleanType()
- .noDefaultValue()
- .withDescription(
- "Expose split host information to use Flink's locality aware split assigner.");
-
- public static final ConfigOption<Integer> SOURCE_READER_FETCH_BATCH_RECORD_COUNT =
- ConfigOptions.key("table.exec.iceberg.fetch-batch-record-count")
- .intType()
- .defaultValue(2048)
- .withDescription("The target number of records for Iceberg reader fetch batch.");
-
- public static final ConfigOption<Integer> TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE =
- ConfigOptions.key("table.exec.iceberg.worker-pool-size")
- .intType()
- .defaultValue(ThreadPools.WORKER_THREAD_POOL_SIZE)
- .withDescription("The size of workers pool used to plan or scan manifests.");
-
- public static final ConfigOption<Boolean> TABLE_EXEC_ICEBERG_USE_FLIP27_SOURCE =
- ConfigOptions.key("table.exec.iceberg.use-flip27-source")
- .booleanType()
- .defaultValue(false)
- .withDescription("Use the FLIP-27 based Iceberg source implementation.");
-
- public static final ConfigOption<SplitAssignerType> TABLE_EXEC_SPLIT_ASSIGNER_TYPE =
- ConfigOptions.key("table.exec.iceberg.split-assigner-type")
- .enumType(SplitAssignerType.class)
- .defaultValue(SplitAssignerType.SIMPLE)
- .withDescription(
- Description.builder()
- .text("Split assigner type that determine how splits are assigned to readers.")
- .linebreak()
- .list(
- TextElement.text(
- SplitAssignerType.SIMPLE
- + ": simple assigner that doesn't provide any guarantee on order or locality."))
- .build());
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
deleted file mode 100644
index 8e1f420b72..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.util.Map;
-import java.util.Set;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.catalog.CatalogBaseTable;
-import org.apache.flink.table.catalog.CatalogDatabaseImpl;
-import org.apache.flink.table.catalog.CatalogTable;
-import org.apache.flink.table.catalog.ObjectIdentifier;
-import org.apache.flink.table.catalog.ObjectPath;
-import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
-import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
-import org.apache.flink.table.connector.sink.DynamicTableSink;
-import org.apache.flink.table.connector.source.DynamicTableSource;
-import org.apache.flink.table.factories.DynamicTableSinkFactory;
-import org.apache.flink.table.factories.DynamicTableSourceFactory;
-import org.apache.flink.table.utils.TableSchemaUtils;
-import org.apache.flink.util.Preconditions;
-import org.apache.iceberg.catalog.TableIdentifier;
-import org.apache.iceberg.exceptions.AlreadyExistsException;
-import org.apache.iceberg.flink.source.IcebergTableSource;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-
-public class FlinkDynamicTableFactory
- implements DynamicTableSinkFactory, DynamicTableSourceFactory {
- static final String FACTORY_IDENTIFIER = "iceberg";
-
- private static final ConfigOption<String> CATALOG_NAME =
- ConfigOptions.key("catalog-name")
- .stringType()
- .noDefaultValue()
- .withDescription("Catalog name");
-
- private static final ConfigOption<String> CATALOG_TYPE =
- ConfigOptions.key(FlinkCatalogFactory.ICEBERG_CATALOG_TYPE)
- .stringType()
- .noDefaultValue()
- .withDescription("Catalog type, the optional types are: custom, hadoop, hive.");
-
- private static final ConfigOption<String> CATALOG_DATABASE =
- ConfigOptions.key("catalog-database")
- .stringType()
- .defaultValue(FlinkCatalogFactory.DEFAULT_DATABASE_NAME)
- .withDescription("Database name managed in the iceberg catalog.");
-
- private static final ConfigOption<String> CATALOG_TABLE =
- ConfigOptions.key("catalog-table")
- .stringType()
- .noDefaultValue()
- .withDescription("Table name managed in the underlying iceberg catalog and database.");
-
- private final FlinkCatalog catalog;
-
- public FlinkDynamicTableFactory() {
- this.catalog = null;
- }
-
- public FlinkDynamicTableFactory(FlinkCatalog catalog) {
- this.catalog = catalog;
- }
-
- @Override
- public DynamicTableSource createDynamicTableSource(Context context) {
- ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
- CatalogTable catalogTable = context.getCatalogTable();
- Map<String, String> tableProps = catalogTable.getOptions();
- TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(catalogTable.getSchema());
-
- TableLoader tableLoader;
- if (catalog != null) {
- tableLoader = createTableLoader(catalog, objectIdentifier.toObjectPath());
- } else {
- tableLoader =
- createTableLoader(
- catalogTable,
- tableProps,
- objectIdentifier.getDatabaseName(),
- objectIdentifier.getObjectName());
- }
-
- return new IcebergTableSource(tableLoader, tableSchema, tableProps, context.getConfiguration());
- }
-
- @Override
- public DynamicTableSink createDynamicTableSink(Context context) {
- ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
- CatalogTable catalogTable = context.getCatalogTable();
- Map<String, String> writeProps = catalogTable.getOptions();
- TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(catalogTable.getSchema());
-
- TableLoader tableLoader;
- if (catalog != null) {
- tableLoader = createTableLoader(catalog, objectIdentifier.toObjectPath());
- } else {
- tableLoader =
- createTableLoader(
- catalogTable,
- writeProps,
- objectIdentifier.getDatabaseName(),
- objectIdentifier.getObjectName());
- }
-
- return new IcebergTableSink(tableLoader, tableSchema, context.getConfiguration(), writeProps);
- }
-
- @Override
- public Set<ConfigOption<?>> requiredOptions() {
- Set<ConfigOption<?>> options = Sets.newHashSet();
- options.add(CATALOG_TYPE);
- options.add(CATALOG_NAME);
- return options;
- }
-
- @Override
- public Set<ConfigOption<?>> optionalOptions() {
- Set<ConfigOption<?>> options = Sets.newHashSet();
- options.add(CATALOG_DATABASE);
- options.add(CATALOG_TABLE);
- return options;
- }
-
- @Override
- public String factoryIdentifier() {
- return FACTORY_IDENTIFIER;
- }
-
- private static TableLoader createTableLoader(
- CatalogBaseTable catalogBaseTable,
- Map<String, String> tableProps,
- String databaseName,
- String tableName) {
- Configuration flinkConf = new Configuration();
- tableProps.forEach(flinkConf::setString);
-
- String catalogName = flinkConf.getString(CATALOG_NAME);
- Preconditions.checkNotNull(
- catalogName, "Table property '%s' cannot be null", CATALOG_NAME.key());
-
- String catalogDatabase = flinkConf.getString(CATALOG_DATABASE, databaseName);
- Preconditions.checkNotNull(catalogDatabase, "The iceberg database name cannot be null");
-
- String catalogTable = flinkConf.getString(CATALOG_TABLE, tableName);
- Preconditions.checkNotNull(catalogTable, "The iceberg table name cannot be null");
-
- org.apache.hadoop.conf.Configuration hadoopConf = FlinkCatalogFactory.clusterHadoopConf();
- FlinkCatalogFactory factory = new FlinkCatalogFactory();
- FlinkCatalog flinkCatalog =
- (FlinkCatalog) factory.createCatalog(catalogName, tableProps, hadoopConf);
- ObjectPath objectPath = new ObjectPath(catalogDatabase, catalogTable);
-
- // Create database if not exists in the external catalog.
- if (!flinkCatalog.databaseExists(catalogDatabase)) {
- try {
- flinkCatalog.createDatabase(
- catalogDatabase, new CatalogDatabaseImpl(Maps.newHashMap(), null), true);
- } catch (DatabaseAlreadyExistException e) {
- throw new AlreadyExistsException(
- e,
- "Database %s already exists in the iceberg catalog %s.",
- catalogName,
- catalogDatabase);
- }
- }
-
- // Create table if not exists in the external catalog.
- if (!flinkCatalog.tableExists(objectPath)) {
- try {
- flinkCatalog.createIcebergTable(objectPath, catalogBaseTable, true);
- } catch (TableAlreadyExistException e) {
- throw new AlreadyExistsException(
- e,
- "Table %s already exists in the database %s and catalog %s",
- catalogTable,
- catalogDatabase,
- catalogName);
- }
- }
-
- return TableLoader.fromCatalog(
- flinkCatalog.getCatalogLoader(), TableIdentifier.of(catalogDatabase, catalogTable));
- }
-
- private static TableLoader createTableLoader(FlinkCatalog catalog, ObjectPath objectPath) {
- Preconditions.checkNotNull(catalog, "Flink catalog cannot be null");
- return TableLoader.fromCatalog(catalog.getCatalogLoader(), catalog.toIdentifier(objectPath));
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java
deleted file mode 100644
index f35bb577fb..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import org.apache.iceberg.EnvironmentContext;
-import org.apache.iceberg.flink.util.FlinkPackage;
-
-class FlinkEnvironmentContext {
- private FlinkEnvironmentContext() {}
-
- public static void init() {
- EnvironmentContext.put(EnvironmentContext.ENGINE_NAME, "flink");
- EnvironmentContext.put(EnvironmentContext.ENGINE_VERSION, FlinkPackage.version());
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java
deleted file mode 100644
index f2244d5137..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkFilters.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.BiFunction;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.flink.table.expressions.CallExpression;
-import org.apache.flink.table.expressions.FieldReferenceExpression;
-import org.apache.flink.table.expressions.ResolvedExpression;
-import org.apache.flink.table.expressions.ValueLiteralExpression;
-import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
-import org.apache.flink.table.functions.FunctionDefinition;
-import org.apache.iceberg.expressions.Expression;
-import org.apache.iceberg.expressions.Expression.Operation;
-import org.apache.iceberg.expressions.Expressions;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.util.DateTimeUtil;
-import org.apache.iceberg.util.NaNUtil;
-
-public class FlinkFilters {
- private FlinkFilters() {}
-
- private static final Pattern STARTS_WITH_PATTERN = Pattern.compile("([^%]+)%");
-
- private static final Map<FunctionDefinition, Operation> FILTERS =
- ImmutableMap.<FunctionDefinition, Operation>builder()
- .put(BuiltInFunctionDefinitions.EQUALS, Operation.EQ)
- .put(BuiltInFunctionDefinitions.NOT_EQUALS, Operation.NOT_EQ)
- .put(BuiltInFunctionDefinitions.GREATER_THAN, Operation.GT)
- .put(BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL, Operation.GT_EQ)
- .put(BuiltInFunctionDefinitions.LESS_THAN, Operation.LT)
- .put(BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL, Operation.LT_EQ)
- .put(BuiltInFunctionDefinitions.IS_NULL, Operation.IS_NULL)
- .put(BuiltInFunctionDefinitions.IS_NOT_NULL, Operation.NOT_NULL)
- .put(BuiltInFunctionDefinitions.AND, Operation.AND)
- .put(BuiltInFunctionDefinitions.OR, Operation.OR)
- .put(BuiltInFunctionDefinitions.NOT, Operation.NOT)
- .put(BuiltInFunctionDefinitions.LIKE, Operation.STARTS_WITH)
- .buildOrThrow();
-
- /**
- * Convert flink expression to iceberg expression.
- *
- * <p>the BETWEEN, NOT_BETWEEN, IN expression will be converted by flink automatically. the
- * BETWEEN will be converted to (GT_EQ AND LT_EQ), the NOT_BETWEEN will be converted to (LT_EQ OR
- * GT_EQ), the IN will be converted to OR, so we do not add the conversion here
- *
- * @param flinkExpression the flink expression
- * @return the iceberg expression
- */
- public static Optional<Expression> convert(
- org.apache.flink.table.expressions.Expression flinkExpression) {
- if (!(flinkExpression instanceof CallExpression)) {
- return Optional.empty();
- }
-
- CallExpression call = (CallExpression) flinkExpression;
- Operation op = FILTERS.get(call.getFunctionDefinition());
- if (op != null) {
- switch (op) {
- case IS_NULL:
- return onlyChildAs(call, FieldReferenceExpression.class)
- .map(FieldReferenceExpression::getName)
- .map(Expressions::isNull);
-
- case NOT_NULL:
- return onlyChildAs(call, FieldReferenceExpression.class)
- .map(FieldReferenceExpression::getName)
- .map(Expressions::notNull);
-
- case LT:
- return convertFieldAndLiteral(Expressions::lessThan, Expressions::greaterThan, call);
-
- case LT_EQ:
- return convertFieldAndLiteral(
- Expressions::lessThanOrEqual, Expressions::greaterThanOrEqual, call);
-
- case GT:
- return convertFieldAndLiteral(Expressions::greaterThan, Expressions::lessThan, call);
-
- case GT_EQ:
- return convertFieldAndLiteral(
- Expressions::greaterThanOrEqual, Expressions::lessThanOrEqual, call);
-
- case EQ:
- return convertFieldAndLiteral(
- (ref, lit) -> {
- if (NaNUtil.isNaN(lit)) {
- return Expressions.isNaN(ref);
- } else {
- return Expressions.equal(ref, lit);
- }
- },
- call);
-
- case NOT_EQ:
- return convertFieldAndLiteral(
- (ref, lit) -> {
- if (NaNUtil.isNaN(lit)) {
- return Expressions.notNaN(ref);
- } else {
- return Expressions.notEqual(ref, lit);
- }
- },
- call);
-
- case NOT:
- return onlyChildAs(call, CallExpression.class)
- .flatMap(FlinkFilters::convert)
- .map(Expressions::not);
-
- case AND:
- return convertLogicExpression(Expressions::and, call);
-
- case OR:
- return convertLogicExpression(Expressions::or, call);
-
- case STARTS_WITH:
- return convertLike(call);
- }
- }
-
- return Optional.empty();
- }
-
- private static <T extends ResolvedExpression> Optional<T> onlyChildAs(
- CallExpression call, Class<T> expectedChildClass) {
- List<ResolvedExpression> children = call.getResolvedChildren();
- if (children.size() != 1) {
- return Optional.empty();
- }
-
- ResolvedExpression child = children.get(0);
- if (!expectedChildClass.isInstance(child)) {
- return Optional.empty();
- }
-
- return Optional.of(expectedChildClass.cast(child));
- }
-
- private static Optional<Expression> convertLike(CallExpression call) {
- List<ResolvedExpression> args = call.getResolvedChildren();
- if (args.size() != 2) {
- return Optional.empty();
- }
-
- org.apache.flink.table.expressions.Expression left = args.get(0);
- org.apache.flink.table.expressions.Expression right = args.get(1);
-
- if (left instanceof FieldReferenceExpression && right instanceof ValueLiteralExpression) {
- String name = ((FieldReferenceExpression) left).getName();
- return convertLiteral((ValueLiteralExpression) right)
- .flatMap(
- lit -> {
- if (lit instanceof String) {
- String pattern = (String) lit;
- Matcher matcher = STARTS_WITH_PATTERN.matcher(pattern);
- // exclude special char of LIKE
- // '_' is the wildcard of the SQL LIKE
- if (!pattern.contains("_") && matcher.matches()) {
- return Optional.of(Expressions.startsWith(name, matcher.group(1)));
- }
- }
-
- return Optional.empty();
- });
- }
-
- return Optional.empty();
- }
-
- private static Optional<Expression> convertLogicExpression(
- BiFunction<Expression, Expression, Expression> function, CallExpression call) {
- List<ResolvedExpression> args = call.getResolvedChildren();
- if (args == null || args.size() != 2) {
- return Optional.empty();
- }
-
- Optional<Expression> left = convert(args.get(0));
- Optional<Expression> right = convert(args.get(1));
- if (left.isPresent() && right.isPresent()) {
- return Optional.of(function.apply(left.get(), right.get()));
- }
-
- return Optional.empty();
- }
-
- private static Optional<Object> convertLiteral(ValueLiteralExpression expression) {
- Optional<?> value =
- expression.getValueAs(
- expression.getOutputDataType().getLogicalType().getDefaultConversion());
- return value.map(
- o -> {
- if (o instanceof LocalDateTime) {
- return DateTimeUtil.microsFromTimestamp((LocalDateTime) o);
- } else if (o instanceof Instant) {
- return DateTimeUtil.microsFromInstant((Instant) o);
- } else if (o instanceof LocalTime) {
- return DateTimeUtil.microsFromTime((LocalTime) o);
- } else if (o instanceof LocalDate) {
- return DateTimeUtil.daysFromDate((LocalDate) o);
- }
-
- return o;
- });
- }
-
- private static Optional<Expression> convertFieldAndLiteral(
- BiFunction<String, Object, Expression> expr, CallExpression call) {
- return convertFieldAndLiteral(expr, expr, call);
- }
-
- private static Optional<Expression> convertFieldAndLiteral(
- BiFunction<String, Object, Expression> convertLR,
- BiFunction<String, Object, Expression> convertRL,
- CallExpression call) {
- List<ResolvedExpression> args = call.getResolvedChildren();
- if (args.size() != 2) {
- return Optional.empty();
- }
-
- org.apache.flink.table.expressions.Expression left = args.get(0);
- org.apache.flink.table.expressions.Expression right = args.get(1);
-
- if (left instanceof FieldReferenceExpression && right instanceof ValueLiteralExpression) {
- String name = ((FieldReferenceExpression) left).getName();
- Optional<Object> lit = convertLiteral((ValueLiteralExpression) right);
- if (lit.isPresent()) {
- return Optional.of(convertLR.apply(name, lit.get()));
- }
- } else if (left instanceof ValueLiteralExpression
- && right instanceof FieldReferenceExpression) {
- Optional<Object> lit = convertLiteral((ValueLiteralExpression) left);
- String name = ((FieldReferenceExpression) right).getName();
- if (lit.isPresent()) {
- return Optional.of(convertRL.apply(name, lit.get()));
- }
- }
-
- return Optional.empty();
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java
deleted file mode 100644
index 767d4497ac..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkFixupTypes.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.types.FixupTypes;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.TypeUtil;
-import org.apache.iceberg.types.Types;
-
-/**
- * The uuid and fixed are converted to the same Flink type. Conversion back can produce only one,
- * which may not be correct.
- */
-class FlinkFixupTypes extends FixupTypes {
-
- private FlinkFixupTypes(Schema referenceSchema) {
- super(referenceSchema);
- }
-
- static Schema fixup(Schema schema, Schema referenceSchema) {
- return new Schema(
- TypeUtil.visit(schema, new FlinkFixupTypes(referenceSchema)).asStructType().fields());
- }
-
- @Override
- protected boolean fixupPrimitive(Type.PrimitiveType type, Type source) {
- if (type instanceof Types.FixedType) {
- int length = ((Types.FixedType) type).length();
- return source.typeId() == Type.TypeID.UUID && length == 16;
- }
- return false;
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java
deleted file mode 100644
index 804a956ec9..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkReadConf.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.time.Duration;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.flink.util.TimeUtils;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.flink.source.StreamingStartingStrategy;
-
-public class FlinkReadConf {
-
- private final FlinkConfParser confParser;
-
- public FlinkReadConf(
- Table table, Map<String, String> readOptions, ReadableConfig readableConfig) {
- this.confParser = new FlinkConfParser(table, readOptions, readableConfig);
- }
-
- public Long snapshotId() {
- return confParser.longConf().option(FlinkReadOptions.SNAPSHOT_ID.key()).parseOptional();
- }
-
- public String tag() {
- return confParser.stringConf().option(FlinkReadOptions.TAG.key()).parseOptional();
- }
-
- public String startTag() {
- return confParser.stringConf().option(FlinkReadOptions.START_TAG.key()).parseOptional();
- }
-
- public String endTag() {
- return confParser.stringConf().option(FlinkReadOptions.END_TAG.key()).parseOptional();
- }
-
- public String branch() {
- return confParser.stringConf().option(FlinkReadOptions.BRANCH.key()).parseOptional();
- }
-
- public boolean caseSensitive() {
- return confParser
- .booleanConf()
- .option(FlinkReadOptions.CASE_SENSITIVE)
- .flinkConfig(FlinkReadOptions.CASE_SENSITIVE_OPTION)
- .defaultValue(FlinkReadOptions.CASE_SENSITIVE_OPTION.defaultValue())
- .parse();
- }
-
- public Long asOfTimestamp() {
- return confParser.longConf().option(FlinkReadOptions.AS_OF_TIMESTAMP.key()).parseOptional();
- }
-
- public StreamingStartingStrategy startingStrategy() {
- return confParser
- .enumConfParser(StreamingStartingStrategy.class)
- .option(FlinkReadOptions.STARTING_STRATEGY)
- .flinkConfig(FlinkReadOptions.STARTING_STRATEGY_OPTION)
- .defaultValue(StreamingStartingStrategy.INCREMENTAL_FROM_LATEST_SNAPSHOT)
- .parse();
- }
-
- public Long startSnapshotTimestamp() {
- return confParser
- .longConf()
- .option(FlinkReadOptions.START_SNAPSHOT_TIMESTAMP.key())
- .parseOptional();
- }
-
- public Long startSnapshotId() {
- return confParser.longConf().option(FlinkReadOptions.START_SNAPSHOT_ID.key()).parseOptional();
- }
-
- public Long endSnapshotId() {
- return confParser.longConf().option(FlinkReadOptions.END_SNAPSHOT_ID.key()).parseOptional();
- }
-
- public long splitSize() {
- return confParser
- .longConf()
- .option(FlinkReadOptions.SPLIT_SIZE)
- .flinkConfig(FlinkReadOptions.SPLIT_SIZE_OPTION)
- .tableProperty(TableProperties.SPLIT_SIZE)
- .defaultValue(TableProperties.SPLIT_SIZE_DEFAULT)
- .parse();
- }
-
- public int splitLookback() {
- return confParser
- .intConf()
- .option(FlinkReadOptions.SPLIT_LOOKBACK)
- .flinkConfig(FlinkReadOptions.SPLIT_LOOKBACK_OPTION)
- .tableProperty(TableProperties.SPLIT_LOOKBACK)
- .defaultValue(TableProperties.SPLIT_LOOKBACK_DEFAULT)
- .parse();
- }
-
- public long splitFileOpenCost() {
- return confParser
- .longConf()
- .option(FlinkReadOptions.SPLIT_FILE_OPEN_COST)
- .flinkConfig(FlinkReadOptions.SPLIT_FILE_OPEN_COST_OPTION)
- .tableProperty(TableProperties.SPLIT_OPEN_FILE_COST)
- .defaultValue(TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT)
- .parse();
- }
-
- public boolean streaming() {
- return confParser
- .booleanConf()
- .option(FlinkReadOptions.STREAMING)
- .flinkConfig(FlinkReadOptions.STREAMING_OPTION)
- .defaultValue(FlinkReadOptions.STREAMING_OPTION.defaultValue())
- .parse();
- }
-
- public Duration monitorInterval() {
- String duration =
- confParser
- .stringConf()
- .option(FlinkReadOptions.MONITOR_INTERVAL)
- .flinkConfig(FlinkReadOptions.MONITOR_INTERVAL_OPTION)
- .defaultValue(FlinkReadOptions.MONITOR_INTERVAL_OPTION.defaultValue())
- .parse();
-
- return TimeUtils.parseDuration(duration);
- }
-
- public boolean includeColumnStats() {
- return confParser
- .booleanConf()
- .option(FlinkReadOptions.INCLUDE_COLUMN_STATS)
- .flinkConfig(FlinkReadOptions.INCLUDE_COLUMN_STATS_OPTION)
- .defaultValue(FlinkReadOptions.INCLUDE_COLUMN_STATS_OPTION.defaultValue())
- .parse();
- }
-
- public int maxPlanningSnapshotCount() {
- return confParser
- .intConf()
- .option(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT)
- .flinkConfig(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT_OPTION)
- .defaultValue(FlinkReadOptions.MAX_PLANNING_SNAPSHOT_COUNT_OPTION.defaultValue())
- .parse();
- }
-
- public String nameMapping() {
- return confParser.stringConf().option(TableProperties.DEFAULT_NAME_MAPPING).parseOptional();
- }
-
- public long limit() {
- return confParser
- .longConf()
- .option(FlinkReadOptions.LIMIT)
- .flinkConfig(FlinkReadOptions.LIMIT_OPTION)
- .defaultValue(FlinkReadOptions.LIMIT_OPTION.defaultValue())
- .parse();
- }
-
- public int workerPoolSize() {
- return confParser
- .intConf()
- .option(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.key())
- .flinkConfig(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE)
- .defaultValue(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.defaultValue())
- .parse();
- }
-
- public int maxAllowedPlanningFailures() {
- return confParser
- .intConf()
- .option(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES)
- .flinkConfig(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES_OPTION)
- .defaultValue(FlinkReadOptions.MAX_ALLOWED_PLANNING_FAILURES_OPTION.defaultValue())
- .parse();
- }
-
- public String watermarkColumn() {
- return confParser
- .stringConf()
- .option(FlinkReadOptions.WATERMARK_COLUMN)
- .flinkConfig(FlinkReadOptions.WATERMARK_COLUMN_OPTION)
- .defaultValue(FlinkReadOptions.WATERMARK_COLUMN_OPTION.defaultValue())
- .parseOptional();
- }
-
- public TimeUnit watermarkColumnTimeUnit() {
- return confParser
- .enumConfParser(TimeUnit.class)
- .option(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT)
- .flinkConfig(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT_OPTION)
- .defaultValue(FlinkReadOptions.WATERMARK_COLUMN_TIME_UNIT_OPTION.defaultValue())
- .parse();
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java
deleted file mode 100644
index 1bbd88146c..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkReadOptions.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.util.concurrent.TimeUnit;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.flink.source.StreamingStartingStrategy;
-
-/** Flink source read options */
-public class FlinkReadOptions {
- private static final String PREFIX = "connector.iceberg.";
-
- private FlinkReadOptions() {}
-
- public static final ConfigOption<Long> SNAPSHOT_ID =
- ConfigOptions.key("snapshot-id").longType().defaultValue(null);
-
- public static final ConfigOption<String> TAG =
- ConfigOptions.key("tag").stringType().defaultValue(null);
-
- public static final ConfigOption<String> BRANCH =
- ConfigOptions.key("branch").stringType().defaultValue(null);
-
- public static final ConfigOption<String> START_TAG =
- ConfigOptions.key("start-tag").stringType().defaultValue(null);
-
- public static final ConfigOption<String> END_TAG =
- ConfigOptions.key("end-tag").stringType().defaultValue(null);
-
- public static final String CASE_SENSITIVE = "case-sensitive";
- public static final ConfigOption<Boolean> CASE_SENSITIVE_OPTION =
- ConfigOptions.key(PREFIX + CASE_SENSITIVE).booleanType().defaultValue(false);
-
- public static final ConfigOption<Long> AS_OF_TIMESTAMP =
- ConfigOptions.key("as-of-timestamp").longType().defaultValue(null);
-
- public static final String STARTING_STRATEGY = "starting-strategy";
- public static final ConfigOption<StreamingStartingStrategy> STARTING_STRATEGY_OPTION =
- ConfigOptions.key(PREFIX + STARTING_STRATEGY)
- .enumType(StreamingStartingStrategy.class)
- .defaultValue(StreamingStartingStrategy.INCREMENTAL_FROM_LATEST_SNAPSHOT);
-
- public static final ConfigOption<Long> START_SNAPSHOT_TIMESTAMP =
- ConfigOptions.key("start-snapshot-timestamp").longType().defaultValue(null);
-
- public static final ConfigOption<Long> START_SNAPSHOT_ID =
- ConfigOptions.key("start-snapshot-id").longType().defaultValue(null);
-
- public static final ConfigOption<Long> END_SNAPSHOT_ID =
- ConfigOptions.key("end-snapshot-id").longType().defaultValue(null);
-
- public static final String SPLIT_SIZE = "split-size";
- public static final ConfigOption<Long> SPLIT_SIZE_OPTION =
- ConfigOptions.key(PREFIX + SPLIT_SIZE)
- .longType()
- .defaultValue(TableProperties.SPLIT_SIZE_DEFAULT);
-
- public static final String SPLIT_LOOKBACK = "split-lookback";
- public static final ConfigOption<Integer> SPLIT_LOOKBACK_OPTION =
- ConfigOptions.key(PREFIX + SPLIT_LOOKBACK)
- .intType()
- .defaultValue(TableProperties.SPLIT_LOOKBACK_DEFAULT);
-
- public static final String SPLIT_FILE_OPEN_COST = "split-file-open-cost";
- public static final ConfigOption<Long> SPLIT_FILE_OPEN_COST_OPTION =
- ConfigOptions.key(PREFIX + SPLIT_FILE_OPEN_COST)
- .longType()
- .defaultValue(TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT);
-
- public static final String STREAMING = "streaming";
- public static final ConfigOption<Boolean> STREAMING_OPTION =
- ConfigOptions.key(PREFIX + STREAMING).booleanType().defaultValue(false);
-
- public static final String MONITOR_INTERVAL = "monitor-interval";
- public static final ConfigOption<String> MONITOR_INTERVAL_OPTION =
- ConfigOptions.key(PREFIX + MONITOR_INTERVAL).stringType().defaultValue("60s");
-
- public static final String INCLUDE_COLUMN_STATS = "include-column-stats";
- public static final ConfigOption<Boolean> INCLUDE_COLUMN_STATS_OPTION =
- ConfigOptions.key(PREFIX + INCLUDE_COLUMN_STATS).booleanType().defaultValue(false);
-
- public static final String MAX_PLANNING_SNAPSHOT_COUNT = "max-planning-snapshot-count";
- public static final ConfigOption<Integer> MAX_PLANNING_SNAPSHOT_COUNT_OPTION =
- ConfigOptions.key(PREFIX + MAX_PLANNING_SNAPSHOT_COUNT)
- .intType()
- .defaultValue(Integer.MAX_VALUE);
-
- public static final String LIMIT = "limit";
- public static final ConfigOption<Long> LIMIT_OPTION =
- ConfigOptions.key(PREFIX + LIMIT).longType().defaultValue(-1L);
-
- public static final String MAX_ALLOWED_PLANNING_FAILURES = "max-allowed-planning-failures";
- public static final ConfigOption<Integer> MAX_ALLOWED_PLANNING_FAILURES_OPTION =
- ConfigOptions.key(PREFIX + MAX_ALLOWED_PLANNING_FAILURES).intType().defaultValue(3);
-
- public static final String WATERMARK_COLUMN = "watermark-column";
- public static final ConfigOption<String> WATERMARK_COLUMN_OPTION =
- ConfigOptions.key(PREFIX + WATERMARK_COLUMN).stringType().noDefaultValue();
-
- public static final String WATERMARK_COLUMN_TIME_UNIT = "watermark-column-time-unit";
- public static final ConfigOption<TimeUnit> WATERMARK_COLUMN_TIME_UNIT_OPTION =
- ConfigOptions.key(PREFIX + WATERMARK_COLUMN_TIME_UNIT)
- .enumType(TimeUnit.class)
- .defaultValue(TimeUnit.MICROSECONDS);
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
deleted file mode 100644
index 25725639c3..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.util.List;
-import java.util.Set;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.utils.TypeConversions;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.TypeUtil;
-import org.apache.iceberg.types.Types;
-
-/**
- * Converter between Flink types and Iceberg type. The conversion is not a 1:1 mapping that not
- * allows back-and-forth conversion. So some information might get lost during the back-and-forth
- * conversion.
- *
- * <p>This inconsistent types:
- *
- * <ul>
- * <li>map Iceberg UUID type to Flink BinaryType(16)
- * <li>map Flink VarCharType(_) and CharType(_) to Iceberg String type
- * <li>map Flink VarBinaryType(_) to Iceberg Binary type
- * <li>map Flink TimeType(_) to Iceberg Time type (microseconds)
- * <li>map Flink TimestampType(_) to Iceberg Timestamp without zone type (microseconds)
- * <li>map Flink LocalZonedTimestampType(_) to Iceberg Timestamp with zone type (microseconds)
- * <li>map Flink MultiSetType to Iceberg Map type(element, int)
- * </ul>
- *
- * <p>
- */
-public class FlinkSchemaUtil {
-
- private FlinkSchemaUtil() {}
-
- /** Convert the flink table schema to apache iceberg schema. */
- public static Schema convert(TableSchema schema) {
- LogicalType schemaType = schema.toRowDataType().getLogicalType();
- Preconditions.checkArgument(
- schemaType instanceof RowType, "Schema logical type should be RowType.");
-
- RowType root = (RowType) schemaType;
- Type converted = root.accept(new FlinkTypeToType(root));
-
- Schema iSchema = new Schema(converted.asStructType().fields());
- return freshIdentifierFieldIds(iSchema, schema);
- }
-
- private static Schema freshIdentifierFieldIds(Schema iSchema, TableSchema schema) {
- // Locate the identifier field id list.
- Set<Integer> identifierFieldIds = Sets.newHashSet();
- if (schema.getPrimaryKey().isPresent()) {
- for (String column : schema.getPrimaryKey().get().getColumns()) {
- Types.NestedField field = iSchema.findField(column);
- Preconditions.checkNotNull(
- field,
- "Cannot find field ID for the primary key column %s in schema %s",
- column,
- iSchema);
- identifierFieldIds.add(field.fieldId());
- }
- }
-
- return new Schema(iSchema.schemaId(), iSchema.asStruct().fields(), identifierFieldIds);
- }
-
- /**
- * Convert a Flink {@link TableSchema} to a {@link Schema} based on the given schema.
- *
- * <p>This conversion does not assign new ids; it uses ids from the base schema.
- *
- * <p>Data types, field order, and nullability will match the Flink type. This conversion may
- * return a schema that is not compatible with base schema.
- *
- * @param baseSchema a Schema on which conversion is based
- * @param flinkSchema a Flink TableSchema
- * @return the equivalent Schema
- * @throws IllegalArgumentException if the type cannot be converted or there are missing ids
- */
- public static Schema convert(Schema baseSchema, TableSchema flinkSchema) {
- // convert to a type with fresh ids
- Types.StructType struct = convert(flinkSchema).asStruct();
- // reassign ids to match the base schema
- Schema schema = TypeUtil.reassignIds(new Schema(struct.fields()), baseSchema);
- // reassign doc to match the base schema
- schema = TypeUtil.reassignDoc(schema, baseSchema);
-
- // fix types that can't be represented in Flink (UUID)
- Schema fixedSchema = FlinkFixupTypes.fixup(schema, baseSchema);
- return freshIdentifierFieldIds(fixedSchema, flinkSchema);
- }
-
- /**
- * Convert a {@link Schema} to a {@link RowType Flink type}.
- *
- * @param schema a Schema
- * @return the equivalent Flink type
- * @throws IllegalArgumentException if the type cannot be converted to Flink
- */
- public static RowType convert(Schema schema) {
- return (RowType) TypeUtil.visit(schema, new TypeToFlinkType());
- }
-
- /**
- * Convert a {@link Type} to a {@link LogicalType Flink type}.
- *
- * @param type a Type
- * @return the equivalent Flink type
- * @throws IllegalArgumentException if the type cannot be converted to Flink
- */
- public static LogicalType convert(Type type) {
- return TypeUtil.visit(type, new TypeToFlinkType());
- }
-
- /**
- * Convert a {@link RowType} to a {@link TableSchema}.
- *
- * @param rowType a RowType
- * @return Flink TableSchema
- */
- public static TableSchema toSchema(RowType rowType) {
- TableSchema.Builder builder = TableSchema.builder();
- for (RowType.RowField field : rowType.getFields()) {
- builder.field(field.getName(), TypeConversions.fromLogicalToDataType(field.getType()));
- }
- return builder.build();
- }
-
- /**
- * Convert a {@link Schema} to a {@link TableSchema}.
- *
- * @param schema iceberg schema to convert.
- * @return Flink TableSchema.
- */
- public static TableSchema toSchema(Schema schema) {
- TableSchema.Builder builder = TableSchema.builder();
-
- // Add columns.
- for (RowType.RowField field : convert(schema).getFields()) {
- builder.field(field.getName(), TypeConversions.fromLogicalToDataType(field.getType()));
- }
-
- // Add primary key.
- Set<Integer> identifierFieldIds = schema.identifierFieldIds();
- if (!identifierFieldIds.isEmpty()) {
- List<String> columns = Lists.newArrayListWithExpectedSize(identifierFieldIds.size());
- for (Integer identifierFieldId : identifierFieldIds) {
- String columnName = schema.findColumnName(identifierFieldId);
- Preconditions.checkNotNull(
- columnName, "Cannot find field with id %s in schema %s", identifierFieldId, schema);
-
- columns.add(columnName);
- }
- builder.primaryKey(columns.toArray(new String[0]));
- }
-
- return builder.build();
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java
deleted file mode 100644
index 5fbd84909d..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkSourceFilter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.expressions.Evaluator;
-import org.apache.iceberg.expressions.Expression;
-import org.apache.iceberg.types.Types;
-
-public class FlinkSourceFilter implements FilterFunction<RowData> {
-
- private final RowType rowType;
- private final Evaluator evaluator;
- private final Types.StructType struct;
- private volatile RowDataWrapper wrapper;
-
- public FlinkSourceFilter(Schema schema, Expression expr, boolean caseSensitive) {
- this.rowType = FlinkSchemaUtil.convert(schema);
- this.struct = schema.asStruct();
- this.evaluator = new Evaluator(struct, expr, caseSensitive);
- }
-
- @Override
- public boolean filter(RowData value) {
- if (wrapper == null) {
- this.wrapper = new RowDataWrapper(rowType, struct);
- }
- return evaluator.eval(wrapper.wrap(value));
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java
deleted file mode 100644
index 6f8bfef2ef..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeToType.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.BigIntType;
-import org.apache.flink.table.types.logical.BinaryType;
-import org.apache.flink.table.types.logical.BooleanType;
-import org.apache.flink.table.types.logical.CharType;
-import org.apache.flink.table.types.logical.DateType;
-import org.apache.flink.table.types.logical.DecimalType;
-import org.apache.flink.table.types.logical.DoubleType;
-import org.apache.flink.table.types.logical.FloatType;
-import org.apache.flink.table.types.logical.IntType;
-import org.apache.flink.table.types.logical.LocalZonedTimestampType;
-import org.apache.flink.table.types.logical.MapType;
-import org.apache.flink.table.types.logical.MultisetType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.logical.SmallIntType;
-import org.apache.flink.table.types.logical.TimeType;
-import org.apache.flink.table.types.logical.TimestampType;
-import org.apache.flink.table.types.logical.TinyIntType;
-import org.apache.flink.table.types.logical.VarBinaryType;
-import org.apache.flink.table.types.logical.VarCharType;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-
-class FlinkTypeToType extends FlinkTypeVisitor<Type> {
-
- private final RowType root;
- private int nextId;
-
- FlinkTypeToType(RowType root) {
- this.root = root;
- // the root struct's fields use the first ids
- this.nextId = root.getFieldCount();
- }
-
- private int getNextId() {
- int next = nextId;
- nextId += 1;
- return next;
- }
-
- @Override
- public Type visit(CharType charType) {
- return Types.StringType.get();
- }
-
- @Override
- public Type visit(VarCharType varCharType) {
- return Types.StringType.get();
- }
-
- @Override
- public Type visit(BooleanType booleanType) {
- return Types.BooleanType.get();
- }
-
- @Override
- public Type visit(BinaryType binaryType) {
- return Types.FixedType.ofLength(binaryType.getLength());
- }
-
- @Override
- public Type visit(VarBinaryType varBinaryType) {
- return Types.BinaryType.get();
- }
-
- @Override
- public Type visit(DecimalType decimalType) {
- return Types.DecimalType.of(decimalType.getPrecision(), decimalType.getScale());
- }
-
- @Override
- public Type visit(TinyIntType tinyIntType) {
- return Types.IntegerType.get();
- }
-
- @Override
- public Type visit(SmallIntType smallIntType) {
- return Types.IntegerType.get();
- }
-
- @Override
- public Type visit(IntType intType) {
- return Types.IntegerType.get();
- }
-
- @Override
- public Type visit(BigIntType bigIntType) {
- return Types.LongType.get();
- }
-
- @Override
- public Type visit(FloatType floatType) {
- return Types.FloatType.get();
- }
-
- @Override
- public Type visit(DoubleType doubleType) {
- return Types.DoubleType.get();
- }
-
- @Override
- public Type visit(DateType dateType) {
- return Types.DateType.get();
- }
-
- @Override
- public Type visit(TimeType timeType) {
- return Types.TimeType.get();
- }
-
- @Override
- public Type visit(TimestampType timestampType) {
- return Types.TimestampType.withoutZone();
- }
-
- @Override
- public Type visit(LocalZonedTimestampType localZonedTimestampType) {
- return Types.TimestampType.withZone();
- }
-
- @Override
- public Type visit(ArrayType arrayType) {
- Type elementType = arrayType.getElementType().accept(this);
- if (arrayType.getElementType().isNullable()) {
- return Types.ListType.ofOptional(getNextId(), elementType);
- } else {
- return Types.ListType.ofRequired(getNextId(), elementType);
- }
- }
-
- @Override
- public Type visit(MultisetType multisetType) {
- Type elementType = multisetType.getElementType().accept(this);
- return Types.MapType.ofRequired(getNextId(), getNextId(), elementType, Types.IntegerType.get());
- }
-
- @Override
- public Type visit(MapType mapType) {
- // keys in map are not allowed to be null.
- Type keyType = mapType.getKeyType().accept(this);
- Type valueType = mapType.getValueType().accept(this);
- if (mapType.getValueType().isNullable()) {
- return Types.MapType.ofOptional(getNextId(), getNextId(), keyType, valueType);
- } else {
- return Types.MapType.ofRequired(getNextId(), getNextId(), keyType, valueType);
- }
- }
-
- @Override
- @SuppressWarnings("ReferenceEquality")
- public Type visit(RowType rowType) {
- List<Types.NestedField> newFields = Lists.newArrayListWithExpectedSize(rowType.getFieldCount());
- boolean isRoot = root == rowType;
-
- List<Type> types =
- rowType.getFields().stream()
- .map(f -> f.getType().accept(this))
- .collect(Collectors.toList());
-
- for (int i = 0; i < rowType.getFieldCount(); i++) {
- int id = isRoot ? i : getNextId();
-
- RowType.RowField field = rowType.getFields().get(i);
- String name = field.getName();
- String comment = field.getDescription().orElse(null);
-
- if (field.getType().isNullable()) {
- newFields.add(Types.NestedField.optional(id, name, types.get(i), comment));
- } else {
- newFields.add(Types.NestedField.required(id, name, types.get(i), comment));
- }
- }
-
- return Types.StructType.of(newFields);
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java
deleted file mode 100644
index f3de241608..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkTypeVisitor.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import org.apache.flink.table.types.logical.DayTimeIntervalType;
-import org.apache.flink.table.types.logical.DistinctType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.LogicalTypeVisitor;
-import org.apache.flink.table.types.logical.NullType;
-import org.apache.flink.table.types.logical.RawType;
-import org.apache.flink.table.types.logical.StructuredType;
-import org.apache.flink.table.types.logical.SymbolType;
-import org.apache.flink.table.types.logical.YearMonthIntervalType;
-import org.apache.flink.table.types.logical.ZonedTimestampType;
-
-public abstract class FlinkTypeVisitor<T> implements LogicalTypeVisitor<T> {
-
- // ------------------------- Unsupported types ------------------------------
-
- @Override
- public T visit(ZonedTimestampType zonedTimestampType) {
- throw new UnsupportedOperationException("Unsupported ZonedTimestampType.");
- }
-
- @Override
- public T visit(YearMonthIntervalType yearMonthIntervalType) {
- throw new UnsupportedOperationException("Unsupported YearMonthIntervalType.");
- }
-
- @Override
- public T visit(DayTimeIntervalType dayTimeIntervalType) {
- throw new UnsupportedOperationException("Unsupported DayTimeIntervalType.");
- }
-
- @Override
- public T visit(DistinctType distinctType) {
- throw new UnsupportedOperationException("Unsupported DistinctType.");
- }
-
- @Override
- public T visit(StructuredType structuredType) {
- throw new UnsupportedOperationException("Unsupported StructuredType.");
- }
-
- @Override
- public T visit(NullType nullType) {
- throw new UnsupportedOperationException("Unsupported NullType.");
- }
-
- @Override
- public T visit(RawType<?> rawType) {
- throw new UnsupportedOperationException("Unsupported RawType.");
- }
-
- @Override
- public T visit(SymbolType<?> symbolType) {
- throw new UnsupportedOperationException("Unsupported SymbolType.");
- }
-
- @Override
- public T visit(LogicalType other) {
- throw new UnsupportedOperationException("Unsupported type: " + other);
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java
deleted file mode 100644
index ca7b1120bc..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.time.Duration;
-import java.util.Map;
-import org.apache.flink.annotation.Experimental;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.iceberg.DistributionMode;
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-
-/**
- * A class for common Iceberg configs for Flink writes.
- *
- * <p>If a config is set at multiple levels, the following order of precedence is used (top to
- * bottom):
- *
- * <ol>
- * <li>Write options
- * <li>flink ReadableConfig
- * <li>Table metadata
- * </ol>
- *
- * The most specific value is set in write options and takes precedence over all other configs. If
- * no write option is provided, this class checks the flink configuration for any overrides. If no
- * applicable value is found in the write options, this class uses the table metadata.
- *
- * <p>Note this class is NOT meant to be serialized.
- */
-public class FlinkWriteConf {
-
- private final FlinkConfParser confParser;
-
- public FlinkWriteConf(
- Table table, Map<String, String> writeOptions, ReadableConfig readableConfig) {
- this.confParser = new FlinkConfParser(table, writeOptions, readableConfig);
- }
-
- public boolean overwriteMode() {
- return confParser
- .booleanConf()
- .option(FlinkWriteOptions.OVERWRITE_MODE.key())
- .flinkConfig(FlinkWriteOptions.OVERWRITE_MODE)
- .defaultValue(FlinkWriteOptions.OVERWRITE_MODE.defaultValue())
- .parse();
- }
-
- public boolean upsertMode() {
- return confParser
- .booleanConf()
- .option(FlinkWriteOptions.WRITE_UPSERT_ENABLED.key())
- .flinkConfig(FlinkWriteOptions.WRITE_UPSERT_ENABLED)
- .tableProperty(TableProperties.UPSERT_ENABLED)
- .defaultValue(TableProperties.UPSERT_ENABLED_DEFAULT)
- .parse();
- }
-
- public FileFormat dataFileFormat() {
- String valueAsString =
- confParser
- .stringConf()
- .option(FlinkWriteOptions.WRITE_FORMAT.key())
- .flinkConfig(FlinkWriteOptions.WRITE_FORMAT)
- .tableProperty(TableProperties.DEFAULT_FILE_FORMAT)
- .defaultValue(TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)
- .parse();
- return FileFormat.fromString(valueAsString);
- }
-
- public long targetDataFileSize() {
- return confParser
- .longConf()
- .option(FlinkWriteOptions.TARGET_FILE_SIZE_BYTES.key())
- .flinkConfig(FlinkWriteOptions.TARGET_FILE_SIZE_BYTES)
- .tableProperty(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES)
- .defaultValue(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT)
- .parse();
- }
-
- public String parquetCompressionCodec() {
- return confParser
- .stringConf()
- .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
- .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
- .tableProperty(TableProperties.PARQUET_COMPRESSION)
- .defaultValue(TableProperties.PARQUET_COMPRESSION_DEFAULT)
- .parse();
- }
-
- public String parquetCompressionLevel() {
- return confParser
- .stringConf()
- .option(FlinkWriteOptions.COMPRESSION_LEVEL.key())
- .flinkConfig(FlinkWriteOptions.COMPRESSION_LEVEL)
- .tableProperty(TableProperties.PARQUET_COMPRESSION_LEVEL)
- .defaultValue(TableProperties.PARQUET_COMPRESSION_LEVEL_DEFAULT)
- .parseOptional();
- }
-
- public String avroCompressionCodec() {
- return confParser
- .stringConf()
- .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
- .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
- .tableProperty(TableProperties.AVRO_COMPRESSION)
- .defaultValue(TableProperties.AVRO_COMPRESSION_DEFAULT)
- .parse();
- }
-
- public String avroCompressionLevel() {
- return confParser
- .stringConf()
- .option(FlinkWriteOptions.COMPRESSION_LEVEL.key())
- .flinkConfig(FlinkWriteOptions.COMPRESSION_LEVEL)
- .tableProperty(TableProperties.AVRO_COMPRESSION_LEVEL)
- .defaultValue(TableProperties.AVRO_COMPRESSION_LEVEL_DEFAULT)
- .parseOptional();
- }
-
- public String orcCompressionCodec() {
- return confParser
- .stringConf()
- .option(FlinkWriteOptions.COMPRESSION_CODEC.key())
- .flinkConfig(FlinkWriteOptions.COMPRESSION_CODEC)
- .tableProperty(TableProperties.ORC_COMPRESSION)
- .defaultValue(TableProperties.ORC_COMPRESSION_DEFAULT)
- .parse();
- }
-
- public String orcCompressionStrategy() {
- return confParser
- .stringConf()
- .option(FlinkWriteOptions.COMPRESSION_STRATEGY.key())
- .flinkConfig(FlinkWriteOptions.COMPRESSION_STRATEGY)
- .tableProperty(TableProperties.ORC_COMPRESSION_STRATEGY)
- .defaultValue(TableProperties.ORC_COMPRESSION_STRATEGY_DEFAULT)
- .parse();
- }
-
- public DistributionMode distributionMode() {
- String modeName =
- confParser
- .stringConf()
- .option(FlinkWriteOptions.DISTRIBUTION_MODE.key())
- .flinkConfig(FlinkWriteOptions.DISTRIBUTION_MODE)
- .tableProperty(TableProperties.WRITE_DISTRIBUTION_MODE)
- .defaultValue(TableProperties.WRITE_DISTRIBUTION_MODE_NONE)
- .parse();
- return DistributionMode.fromName(modeName);
- }
-
- public int workerPoolSize() {
- return confParser
- .intConf()
- .flinkConfig(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE)
- .defaultValue(FlinkConfigOptions.TABLE_EXEC_ICEBERG_WORKER_POOL_SIZE.defaultValue())
- .parse();
- }
-
- public String branch() {
- return confParser
- .stringConf()
- .option(FlinkWriteOptions.BRANCH.key())
- .defaultValue(FlinkWriteOptions.BRANCH.defaultValue())
- .parse();
- }
-
- public Integer writeParallelism() {
- return confParser.intConf().option(FlinkWriteOptions.WRITE_PARALLELISM.key()).parseOptional();
- }
-
- /**
- * NOTE: This may be removed or changed in a future release. This value specifies the interval for
- * refreshing the table instances in sink writer subtasks. If not specified then the default
- * behavior is to not refresh the table.
- *
- * @return the interval for refreshing the table in sink writer subtasks
- */
- @Experimental
- public Duration tableRefreshInterval() {
- return confParser
- .durationConf()
- .option(FlinkWriteOptions.TABLE_REFRESH_INTERVAL.key())
- .flinkConfig(FlinkWriteOptions.TABLE_REFRESH_INTERVAL)
- .parseOptional();
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java
deleted file mode 100644
index df73f2e09c..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.time.Duration;
-import org.apache.flink.annotation.Experimental;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.iceberg.SnapshotRef;
-
-/** Flink sink write options */
-public class FlinkWriteOptions {
-
- private FlinkWriteOptions() {}
-
- // File format for write operations(default: Table write.format.default )
- public static final ConfigOption<String> WRITE_FORMAT =
- ConfigOptions.key("write-format").stringType().noDefaultValue();
-
- // Overrides this table's write.target-file-size-bytes
- public static final ConfigOption<Long> TARGET_FILE_SIZE_BYTES =
- ConfigOptions.key("target-file-size-bytes").longType().noDefaultValue();
-
- // Overrides this table's write.<FILE_FORMAT>.compression-codec
- public static final ConfigOption<String> COMPRESSION_CODEC =
- ConfigOptions.key("compression-codec").stringType().noDefaultValue();
-
- // Overrides this table's write.<FILE_FORMAT>.compression-level
- public static final ConfigOption<String> COMPRESSION_LEVEL =
- ConfigOptions.key("compression-level").stringType().noDefaultValue();
-
- // Overrides this table's write.<FILE_FORMAT>.compression-strategy
- public static final ConfigOption<String> COMPRESSION_STRATEGY =
- ConfigOptions.key("compression-strategy").stringType().noDefaultValue();
-
- // Overrides this table's write.upsert.enabled
- public static final ConfigOption<Boolean> WRITE_UPSERT_ENABLED =
- ConfigOptions.key("upsert-enabled").booleanType().noDefaultValue();
-
- public static final ConfigOption<Boolean> OVERWRITE_MODE =
- ConfigOptions.key("overwrite-enabled").booleanType().defaultValue(false);
-
- // Overrides the table's write.distribution-mode
- public static final ConfigOption<String> DISTRIBUTION_MODE =
- ConfigOptions.key("distribution-mode").stringType().noDefaultValue();
-
- // Branch to write to
- public static final ConfigOption<String> BRANCH =
- ConfigOptions.key("branch").stringType().defaultValue(SnapshotRef.MAIN_BRANCH);
-
- public static final ConfigOption<Integer> WRITE_PARALLELISM =
- ConfigOptions.key("write-parallelism").intType().noDefaultValue();
-
- @Experimental
- public static final ConfigOption<Duration> TABLE_REFRESH_INTERVAL =
- ConfigOptions.key("table-refresh-interval").durationType().noDefaultValue();
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java
deleted file mode 100644
index 1b9268569d..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/IcebergTableSink.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.util.List;
-import java.util.Map;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSink;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.api.constraints.UniqueConstraint;
-import org.apache.flink.table.connector.ChangelogMode;
-import org.apache.flink.table.connector.ProviderContext;
-import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
-import org.apache.flink.table.connector.sink.DynamicTableSink;
-import org.apache.flink.table.connector.sink.abilities.SupportsOverwrite;
-import org.apache.flink.table.connector.sink.abilities.SupportsPartitioning;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.types.RowKind;
-import org.apache.flink.util.Preconditions;
-import org.apache.iceberg.flink.sink.FlinkSink;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-
-public class IcebergTableSink implements DynamicTableSink, SupportsPartitioning, SupportsOverwrite {
- private final TableLoader tableLoader;
- private final TableSchema tableSchema;
- private final ReadableConfig readableConfig;
- private final Map<String, String> writeProps;
-
- private boolean overwrite = false;
-
- private IcebergTableSink(IcebergTableSink toCopy) {
- this.tableLoader = toCopy.tableLoader;
- this.tableSchema = toCopy.tableSchema;
- this.overwrite = toCopy.overwrite;
- this.readableConfig = toCopy.readableConfig;
- this.writeProps = toCopy.writeProps;
- }
-
- public IcebergTableSink(
- TableLoader tableLoader,
- TableSchema tableSchema,
- ReadableConfig readableConfig,
- Map<String, String> writeProps) {
- this.tableLoader = tableLoader;
- this.tableSchema = tableSchema;
- this.readableConfig = readableConfig;
- this.writeProps = writeProps;
- }
-
- @Override
- public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
- Preconditions.checkState(
- !overwrite || context.isBounded(),
- "Unbounded data stream doesn't support overwrite operation.");
-
- List<String> equalityColumns =
- tableSchema.getPrimaryKey().map(UniqueConstraint::getColumns).orElseGet(ImmutableList::of);
-
- return new DataStreamSinkProvider() {
- @Override
- public DataStreamSink<?> consumeDataStream(
- ProviderContext providerContext, DataStream<RowData> dataStream) {
- return FlinkSink.forRowData(dataStream)
- .tableLoader(tableLoader)
- .tableSchema(tableSchema)
- .equalityFieldColumns(equalityColumns)
- .overwrite(overwrite)
- .setAll(writeProps)
- .flinkConf(readableConfig)
- .append();
- }
- };
- }
-
- @Override
- public void applyStaticPartition(Map<String, String> partition) {
- // The flink's PartitionFanoutWriter will handle the static partition write policy
- // automatically.
- }
-
- @Override
- public ChangelogMode getChangelogMode(ChangelogMode requestedMode) {
- ChangelogMode.Builder builder = ChangelogMode.newBuilder();
- for (RowKind kind : requestedMode.getContainedKinds()) {
- builder.addContainedKind(kind);
- }
- return builder.build();
- }
-
- @Override
- public DynamicTableSink copy() {
- return new IcebergTableSink(this);
- }
-
- @Override
- public String asSummaryString() {
- return "Iceberg table sink";
- }
-
- @Override
- public void applyOverwrite(boolean newOverwrite) {
- this.overwrite = newOverwrite;
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java
deleted file mode 100644
index d4cec7a3e8..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/RowDataWrapper.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.lang.reflect.Array;
-import java.nio.ByteBuffer;
-import java.time.LocalDateTime;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.types.logical.DecimalType;
-import org.apache.flink.table.types.logical.LocalZonedTimestampType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.logical.TimestampType;
-import org.apache.iceberg.StructLike;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-import org.apache.iceberg.util.DateTimeUtil;
-import org.apache.iceberg.util.UUIDUtil;
-
-public class RowDataWrapper implements StructLike {
-
- private final LogicalType[] types;
- private final PositionalGetter<?>[] getters;
- private RowData rowData = null;
-
- public RowDataWrapper(RowType rowType, Types.StructType struct) {
- int size = rowType.getFieldCount();
-
- types = (LogicalType[]) Array.newInstance(LogicalType.class, size);
- getters = (PositionalGetter[]) Array.newInstance(PositionalGetter.class, size);
-
- for (int i = 0; i < size; i++) {
- types[i] = rowType.getTypeAt(i);
- getters[i] = buildGetter(types[i], struct.fields().get(i).type());
- }
- }
-
- public RowDataWrapper wrap(RowData data) {
- this.rowData = data;
- return this;
- }
-
- @Override
- public int size() {
- return types.length;
- }
-
- @Override
- public <T> T get(int pos, Class<T> javaClass) {
- if (rowData.isNullAt(pos)) {
- return null;
- } else if (getters[pos] != null) {
- return javaClass.cast(getters[pos].get(rowData, pos));
- }
-
- Object value = RowData.createFieldGetter(types[pos], pos).getFieldOrNull(rowData);
- return javaClass.cast(value);
- }
-
- @Override
- public <T> void set(int pos, T value) {
- throw new UnsupportedOperationException(
- "Could not set a field in the RowDataWrapper because rowData is read-only");
- }
-
- private interface PositionalGetter<T> {
- T get(RowData data, int pos);
- }
-
- private static PositionalGetter<?> buildGetter(LogicalType logicalType, Type type) {
- switch (logicalType.getTypeRoot()) {
- case TINYINT:
- return (row, pos) -> (int) row.getByte(pos);
- case SMALLINT:
- return (row, pos) -> (int) row.getShort(pos);
- case CHAR:
- case VARCHAR:
- return (row, pos) -> row.getString(pos).toString();
-
- case BINARY:
- case VARBINARY:
- if (Type.TypeID.UUID == type.typeId()) {
- return (row, pos) -> UUIDUtil.convert(row.getBinary(pos));
- } else {
- return (row, pos) -> ByteBuffer.wrap(row.getBinary(pos));
- }
-
- case DECIMAL:
- DecimalType decimalType = (DecimalType) logicalType;
- return (row, pos) ->
- row.getDecimal(pos, decimalType.getPrecision(), decimalType.getScale()).toBigDecimal();
-
- case TIME_WITHOUT_TIME_ZONE:
- // Time in RowData is in milliseconds (Integer), while iceberg's time is microseconds
- // (Long).
- return (row, pos) -> ((long) row.getInt(pos)) * 1_000;
-
- case TIMESTAMP_WITHOUT_TIME_ZONE:
- TimestampType timestampType = (TimestampType) logicalType;
- return (row, pos) -> {
- LocalDateTime localDateTime =
- row.getTimestamp(pos, timestampType.getPrecision()).toLocalDateTime();
- return DateTimeUtil.microsFromTimestamp(localDateTime);
- };
-
- case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- LocalZonedTimestampType lzTs = (LocalZonedTimestampType) logicalType;
- return (row, pos) -> {
- TimestampData timestampData = row.getTimestamp(pos, lzTs.getPrecision());
- return timestampData.getMillisecond() * 1000
- + timestampData.getNanoOfMillisecond() / 1000;
- };
-
- case ROW:
- RowType rowType = (RowType) logicalType;
- Types.StructType structType = (Types.StructType) type;
-
- RowDataWrapper nestedWrapper = new RowDataWrapper(rowType, structType);
- return (row, pos) -> nestedWrapper.wrap(row.getRow(pos, rowType.getFieldCount()));
-
- default:
- return null;
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java
deleted file mode 100644
index da509451fe..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.Serializable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.catalog.Catalog;
-import org.apache.iceberg.catalog.TableIdentifier;
-import org.apache.iceberg.hadoop.HadoopTables;
-import org.apache.iceberg.hadoop.SerializableConfiguration;
-import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
-
-/**
- * Serializable loader to load an Iceberg {@link Table}. Flink needs to get {@link Table} objects in
- * the cluster (for example, to get splits), not just on the client side. So we need an Iceberg
- * table loader to get the {@link Table} object.
- */
-public interface TableLoader extends Closeable, Serializable, Cloneable {
-
- void open();
-
- boolean isOpen();
-
- Table loadTable();
-
- /** Clone a TableLoader */
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- TableLoader clone();
-
- static TableLoader fromCatalog(CatalogLoader catalogLoader, TableIdentifier identifier) {
- return new CatalogTableLoader(catalogLoader, identifier);
- }
-
- static TableLoader fromHadoopTable(String location) {
- return fromHadoopTable(location, FlinkCatalogFactory.clusterHadoopConf());
- }
-
- static TableLoader fromHadoopTable(String location, Configuration hadoopConf) {
- return new HadoopTableLoader(location, hadoopConf);
- }
-
- class HadoopTableLoader implements TableLoader {
-
- private static final long serialVersionUID = 1L;
-
- private final String location;
- private final SerializableConfiguration hadoopConf;
-
- private transient HadoopTables tables;
-
- private HadoopTableLoader(String location, Configuration conf) {
- this.location = location;
- this.hadoopConf = new SerializableConfiguration(conf);
- }
-
- @Override
- public void open() {
- tables = new HadoopTables(hadoopConf.get());
- }
-
- @Override
- public boolean isOpen() {
- return tables != null;
- }
-
- @Override
- public Table loadTable() {
- FlinkEnvironmentContext.init();
- return tables.load(location);
- }
-
- @Override
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- public TableLoader clone() {
- return new HadoopTableLoader(location, new Configuration(hadoopConf.get()));
- }
-
- @Override
- public void close() {}
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this).add("location", location).toString();
- }
- }
-
- class CatalogTableLoader implements TableLoader {
-
- private static final long serialVersionUID = 1L;
-
- private final CatalogLoader catalogLoader;
- private final String identifier;
-
- private transient Catalog catalog;
-
- private CatalogTableLoader(CatalogLoader catalogLoader, TableIdentifier tableIdentifier) {
- this.catalogLoader = catalogLoader;
- this.identifier = tableIdentifier.toString();
- }
-
- @Override
- public void open() {
- catalog = catalogLoader.loadCatalog();
- }
-
- @Override
- public boolean isOpen() {
- return catalog != null;
- }
-
- @Override
- public Table loadTable() {
- FlinkEnvironmentContext.init();
- return catalog.loadTable(TableIdentifier.parse(identifier));
- }
-
- @Override
- public void close() throws IOException {
- if (catalog instanceof Closeable) {
- ((Closeable) catalog).close();
- }
-
- catalog = null;
- }
-
- @Override
- @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
- public TableLoader clone() {
- return new CatalogTableLoader(catalogLoader.clone(), TableIdentifier.parse(identifier));
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("tableIdentifier", identifier)
- .add("catalogLoader", catalogLoader)
- .toString();
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java
deleted file mode 100644
index f8f1b74b1c..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/TypeToFlinkType.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink;
-
-import java.util.List;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.BigIntType;
-import org.apache.flink.table.types.logical.BinaryType;
-import org.apache.flink.table.types.logical.BooleanType;
-import org.apache.flink.table.types.logical.DateType;
-import org.apache.flink.table.types.logical.DecimalType;
-import org.apache.flink.table.types.logical.DoubleType;
-import org.apache.flink.table.types.logical.FloatType;
-import org.apache.flink.table.types.logical.IntType;
-import org.apache.flink.table.types.logical.LocalZonedTimestampType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.MapType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.logical.TimeType;
-import org.apache.flink.table.types.logical.TimestampType;
-import org.apache.flink.table.types.logical.VarBinaryType;
-import org.apache.flink.table.types.logical.VarCharType;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.TypeUtil;
-import org.apache.iceberg.types.Types;
-
-class TypeToFlinkType extends TypeUtil.SchemaVisitor<LogicalType> {
- TypeToFlinkType() {}
-
- @Override
- public LogicalType schema(Schema schema, LogicalType structType) {
- return structType;
- }
-
- @Override
- public LogicalType struct(Types.StructType struct, List<LogicalType> fieldResults) {
- List<Types.NestedField> fields = struct.fields();
-
- List<RowType.RowField> flinkFields = Lists.newArrayListWithExpectedSize(fieldResults.size());
- for (int i = 0; i < fields.size(); i += 1) {
- Types.NestedField field = fields.get(i);
- LogicalType type = fieldResults.get(i);
- RowType.RowField flinkField =
- new RowType.RowField(field.name(), type.copy(field.isOptional()), field.doc());
- flinkFields.add(flinkField);
- }
-
- return new RowType(flinkFields);
- }
-
- @Override
- public LogicalType field(Types.NestedField field, LogicalType fieldResult) {
- return fieldResult;
- }
-
- @Override
- public LogicalType list(Types.ListType list, LogicalType elementResult) {
- return new ArrayType(elementResult.copy(list.isElementOptional()));
- }
-
- @Override
- public LogicalType map(Types.MapType map, LogicalType keyResult, LogicalType valueResult) {
- // keys in map are not allowed to be null.
- return new MapType(keyResult.copy(false), valueResult.copy(map.isValueOptional()));
- }
-
- @Override
- public LogicalType primitive(Type.PrimitiveType primitive) {
- switch (primitive.typeId()) {
- case BOOLEAN:
- return new BooleanType();
- case INTEGER:
- return new IntType();
- case LONG:
- return new BigIntType();
- case FLOAT:
- return new FloatType();
- case DOUBLE:
- return new DoubleType();
- case DATE:
- return new DateType();
- case TIME:
- // For the type: Flink only support TimeType with default precision (second) now. The
- // precision of time is
- // not supported in Flink, so we can think of it as a simple time type directly.
- // For the data: Flink uses int that support mills to represent time data, so it supports
- // mills precision.
- return new TimeType();
- case TIMESTAMP:
- Types.TimestampType timestamp = (Types.TimestampType) primitive;
- if (timestamp.shouldAdjustToUTC()) {
- // MICROS
- return new LocalZonedTimestampType(6);
- } else {
- // MICROS
- return new TimestampType(6);
- }
- case STRING:
- return new VarCharType(VarCharType.MAX_LENGTH);
- case UUID:
- // UUID length is 16
- return new BinaryType(16);
- case FIXED:
- Types.FixedType fixedType = (Types.FixedType) primitive;
- return new BinaryType(fixedType.length());
- case BINARY:
- return new VarBinaryType(VarBinaryType.MAX_LENGTH);
- case DECIMAL:
- Types.DecimalType decimal = (Types.DecimalType) primitive;
- return new DecimalType(decimal.precision(), decimal.scale());
- default:
- throw new UnsupportedOperationException(
- "Cannot convert unknown type to Flink: " + primitive);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java
deleted file mode 100644
index 06ac54617a..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/actions/Actions.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.actions;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.CoreOptions;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.iceberg.Table;
-
-public class Actions {
-
- public static final Configuration CONFIG =
- new Configuration()
- // disable classloader check as Avro may cache class/object in the serializers.
- .set(CoreOptions.CHECK_LEAKED_CLASSLOADER, false);
-
- private StreamExecutionEnvironment env;
- private Table table;
-
- private Actions(StreamExecutionEnvironment env, Table table) {
- this.env = env;
- this.table = table;
- }
-
- public static Actions forTable(StreamExecutionEnvironment env, Table table) {
- return new Actions(env, table);
- }
-
- public static Actions forTable(Table table) {
- return new Actions(StreamExecutionEnvironment.getExecutionEnvironment(CONFIG), table);
- }
-
- public RewriteDataFilesAction rewriteDataFiles() {
- return new RewriteDataFilesAction(env, table);
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java
deleted file mode 100644
index 9876bb3861..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/actions/RewriteDataFilesAction.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.actions;
-
-import java.util.List;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.iceberg.CombinedScanTask;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.actions.BaseRewriteDataFilesAction;
-import org.apache.iceberg.flink.source.RowDataRewriter;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-
-public class RewriteDataFilesAction extends BaseRewriteDataFilesAction<RewriteDataFilesAction> {
-
- private StreamExecutionEnvironment env;
- private int maxParallelism;
-
- public RewriteDataFilesAction(StreamExecutionEnvironment env, Table table) {
- super(table);
- this.env = env;
- this.maxParallelism = env.getParallelism();
- }
-
- @Override
- protected FileIO fileIO() {
- return table().io();
- }
-
- @Override
- protected List<DataFile> rewriteDataForTasks(List<CombinedScanTask> combinedScanTasks) {
- int size = combinedScanTasks.size();
- int parallelism = Math.min(size, maxParallelism);
- DataStream<CombinedScanTask> dataStream = env.fromCollection(combinedScanTasks);
- RowDataRewriter rowDataRewriter =
- new RowDataRewriter(table(), caseSensitive(), fileIO(), encryptionManager());
- try {
- return rowDataRewriter.rewriteDataForTasks(dataStream, parallelism);
- } catch (Exception e) {
- throw new RuntimeException("Rewrite data file error.", e);
- }
- }
-
- @Override
- protected RewriteDataFilesAction self() {
- return this;
- }
-
- public RewriteDataFilesAction maxParallelism(int parallelism) {
- Preconditions.checkArgument(parallelism > 0, "Invalid max parallelism %s", parallelism);
- this.maxParallelism = parallelism;
- return this;
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java
deleted file mode 100644
index 8103224a0b..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/AvroWithFlinkSchemaVisitor.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.LogicalTypeFamily;
-import org.apache.flink.table.types.logical.MapType;
-import org.apache.flink.table.types.logical.NullType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.iceberg.avro.AvroWithPartnerByStructureVisitor;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.util.Pair;
-
-public abstract class AvroWithFlinkSchemaVisitor<T>
- extends AvroWithPartnerByStructureVisitor<LogicalType, T> {
-
- @Override
- protected boolean isStringType(LogicalType logicalType) {
- return logicalType.getTypeRoot().getFamilies().contains(LogicalTypeFamily.CHARACTER_STRING);
- }
-
- @Override
- protected boolean isMapType(LogicalType logicalType) {
- return logicalType instanceof MapType;
- }
-
- @Override
- protected LogicalType arrayElementType(LogicalType arrayType) {
- Preconditions.checkArgument(
- arrayType instanceof ArrayType, "Invalid array: %s is not an array", arrayType);
- return ((ArrayType) arrayType).getElementType();
- }
-
- @Override
- protected LogicalType mapKeyType(LogicalType mapType) {
- Preconditions.checkArgument(isMapType(mapType), "Invalid map: %s is not a map", mapType);
- return ((MapType) mapType).getKeyType();
- }
-
- @Override
- protected LogicalType mapValueType(LogicalType mapType) {
- Preconditions.checkArgument(isMapType(mapType), "Invalid map: %s is not a map", mapType);
- return ((MapType) mapType).getValueType();
- }
-
- @Override
- protected Pair<String, LogicalType> fieldNameAndType(LogicalType structType, int pos) {
- Preconditions.checkArgument(
- structType instanceof RowType, "Invalid struct: %s is not a struct", structType);
- RowType.RowField field = ((RowType) structType).getFields().get(pos);
- return Pair.of(field.getName(), field.getType());
- }
-
- @Override
- protected LogicalType nullType() {
- return new NullType();
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroReader.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroReader.java
deleted file mode 100644
index 8640495973..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroReader.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Supplier;
-import org.apache.avro.LogicalType;
-import org.apache.avro.LogicalTypes;
-import org.apache.avro.Schema;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.Decoder;
-import org.apache.flink.table.data.RowData;
-import org.apache.iceberg.avro.AvroSchemaWithTypeVisitor;
-import org.apache.iceberg.avro.SupportsRowPosition;
-import org.apache.iceberg.avro.ValueReader;
-import org.apache.iceberg.avro.ValueReaders;
-import org.apache.iceberg.data.avro.DecoderResolver;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-
-public class FlinkAvroReader implements DatumReader<RowData>, SupportsRowPosition {
-
- private final Schema readSchema;
- private final ValueReader<RowData> reader;
- private Schema fileSchema = null;
-
- public FlinkAvroReader(org.apache.iceberg.Schema expectedSchema, Schema readSchema) {
- this(expectedSchema, readSchema, ImmutableMap.of());
- }
-
- @SuppressWarnings("unchecked")
- public FlinkAvroReader(
- org.apache.iceberg.Schema expectedSchema, Schema readSchema, Map<Integer, ?> constants) {
- this.readSchema = readSchema;
- this.reader =
- (ValueReader<RowData>)
- AvroSchemaWithTypeVisitor.visit(expectedSchema, readSchema, new ReadBuilder(constants));
- }
-
- @Override
- public void setSchema(Schema newFileSchema) {
- this.fileSchema = Schema.applyAliases(newFileSchema, readSchema);
- }
-
- @Override
- public RowData read(RowData reuse, Decoder decoder) throws IOException {
- return DecoderResolver.resolveAndRead(decoder, readSchema, fileSchema, reader, reuse);
- }
-
- @Override
- public void setRowPositionSupplier(Supplier<Long> posSupplier) {
- if (reader instanceof SupportsRowPosition) {
- ((SupportsRowPosition) reader).setRowPositionSupplier(posSupplier);
- }
- }
-
- private static class ReadBuilder extends AvroSchemaWithTypeVisitor<ValueReader<?>> {
- private final Map<Integer, ?> idToConstant;
-
- private ReadBuilder(Map<Integer, ?> idToConstant) {
- this.idToConstant = idToConstant;
- }
-
- @Override
- public ValueReader<?> record(
- Types.StructType expected, Schema record, List<String> names, List<ValueReader<?>> fields) {
- return FlinkValueReaders.struct(fields, expected.asStructType(), idToConstant);
- }
-
- @Override
- public ValueReader<?> union(Type expected, Schema union, List<ValueReader<?>> options) {
- return ValueReaders.union(options);
- }
-
- @Override
- public ValueReader<?> array(
- Types.ListType expected, Schema array, ValueReader<?> elementReader) {
- return FlinkValueReaders.array(elementReader);
- }
-
- @Override
- public ValueReader<?> map(
- Types.MapType expected, Schema map, ValueReader<?> keyReader, ValueReader<?> valueReader) {
- return FlinkValueReaders.arrayMap(keyReader, valueReader);
- }
-
- @Override
- public ValueReader<?> map(Types.MapType expected, Schema map, ValueReader<?> valueReader) {
- return FlinkValueReaders.map(FlinkValueReaders.strings(), valueReader);
- }
-
- @Override
- public ValueReader<?> primitive(Type.PrimitiveType expected, Schema primitive) {
- LogicalType logicalType = primitive.getLogicalType();
- if (logicalType != null) {
- switch (logicalType.getName()) {
- case "date":
- return ValueReaders.ints();
-
- case "time-micros":
- return FlinkValueReaders.timeMicros();
-
- case "timestamp-millis":
- return FlinkValueReaders.timestampMills();
-
- case "timestamp-micros":
- return FlinkValueReaders.timestampMicros();
-
- case "decimal":
- LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
- return FlinkValueReaders.decimal(
- ValueReaders.decimalBytesReader(primitive),
- decimal.getPrecision(),
- decimal.getScale());
-
- case "uuid":
- return FlinkValueReaders.uuids();
-
- default:
- throw new IllegalArgumentException("Unknown logical type: " + logicalType);
- }
- }
-
- switch (primitive.getType()) {
- case NULL:
- return ValueReaders.nulls();
- case BOOLEAN:
- return ValueReaders.booleans();
- case INT:
- return ValueReaders.ints();
- case LONG:
- return ValueReaders.longs();
- case FLOAT:
- return ValueReaders.floats();
- case DOUBLE:
- return ValueReaders.doubles();
- case STRING:
- return FlinkValueReaders.strings();
- case FIXED:
- return ValueReaders.fixed(primitive.getFixedSize());
- case BYTES:
- return ValueReaders.bytes();
- case ENUM:
- return FlinkValueReaders.enums(primitive.getEnumSymbols());
- default:
- throw new IllegalArgumentException("Unsupported type: " + primitive);
- }
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java
deleted file mode 100644
index 873e657831..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkAvroWriter.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.avro.LogicalTypes;
-import org.apache.avro.Schema;
-import org.apache.avro.io.Encoder;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.iceberg.FieldMetrics;
-import org.apache.iceberg.avro.MetricsAwareDatumWriter;
-import org.apache.iceberg.avro.ValueWriter;
-import org.apache.iceberg.avro.ValueWriters;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-
-public class FlinkAvroWriter implements MetricsAwareDatumWriter<RowData> {
- private final RowType rowType;
- private ValueWriter<RowData> writer = null;
-
- public FlinkAvroWriter(RowType rowType) {
- this.rowType = rowType;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void setSchema(Schema schema) {
- this.writer =
- (ValueWriter<RowData>)
- AvroWithFlinkSchemaVisitor.visit(rowType, schema, new WriteBuilder());
- }
-
- @Override
- public void write(RowData datum, Encoder out) throws IOException {
- writer.write(datum, out);
- }
-
- @Override
- public Stream<FieldMetrics> metrics() {
- return writer.metrics();
- }
-
- private static class WriteBuilder extends AvroWithFlinkSchemaVisitor<ValueWriter<?>> {
- @Override
- public ValueWriter<?> record(
- LogicalType struct, Schema record, List<String> names, List<ValueWriter<?>> fields) {
- return FlinkValueWriters.row(
- fields,
- IntStream.range(0, names.size())
- .mapToObj(i -> fieldNameAndType(struct, i).second())
- .collect(Collectors.toList()));
- }
-
- @Override
- public ValueWriter<?> union(LogicalType type, Schema union, List<ValueWriter<?>> options) {
- Preconditions.checkArgument(
- options.contains(ValueWriters.nulls()),
- "Cannot create writer for non-option union: %s",
- union);
- Preconditions.checkArgument(
- options.size() == 2, "Cannot create writer for non-option union: %s", union);
- if (union.getTypes().get(0).getType() == Schema.Type.NULL) {
- return ValueWriters.option(0, options.get(1));
- } else {
- return ValueWriters.option(1, options.get(0));
- }
- }
-
- @Override
- public ValueWriter<?> array(LogicalType sArray, Schema array, ValueWriter<?> elementWriter) {
- return FlinkValueWriters.array(elementWriter, arrayElementType(sArray));
- }
-
- @Override
- public ValueWriter<?> map(LogicalType sMap, Schema map, ValueWriter<?> valueReader) {
- return FlinkValueWriters.map(
- FlinkValueWriters.strings(), mapKeyType(sMap), valueReader, mapValueType(sMap));
- }
-
- @Override
- public ValueWriter<?> map(
- LogicalType sMap, Schema map, ValueWriter<?> keyWriter, ValueWriter<?> valueWriter) {
- return FlinkValueWriters.arrayMap(
- keyWriter, mapKeyType(sMap), valueWriter, mapValueType(sMap));
- }
-
- @Override
- public ValueWriter<?> primitive(LogicalType type, Schema primitive) {
- org.apache.avro.LogicalType logicalType = primitive.getLogicalType();
- if (logicalType != null) {
- switch (logicalType.getName()) {
- case "date":
- return ValueWriters.ints();
-
- case "time-micros":
- return FlinkValueWriters.timeMicros();
-
- case "timestamp-micros":
- return FlinkValueWriters.timestampMicros();
-
- case "decimal":
- LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
- return FlinkValueWriters.decimal(decimal.getPrecision(), decimal.getScale());
-
- case "uuid":
- return ValueWriters.uuids();
-
- default:
- throw new IllegalArgumentException("Unsupported logical type: " + logicalType);
- }
- }
-
- switch (primitive.getType()) {
- case NULL:
- return ValueWriters.nulls();
- case BOOLEAN:
- return ValueWriters.booleans();
- case INT:
- switch (type.getTypeRoot()) {
- case TINYINT:
- return ValueWriters.tinyints();
- case SMALLINT:
- return ValueWriters.shorts();
- default:
- return ValueWriters.ints();
- }
- case LONG:
- return ValueWriters.longs();
- case FLOAT:
- return ValueWriters.floats();
- case DOUBLE:
- return ValueWriters.doubles();
- case STRING:
- return FlinkValueWriters.strings();
- case FIXED:
- return ValueWriters.fixed(primitive.getFixedSize());
- case BYTES:
- return ValueWriters.bytes();
- default:
- throw new IllegalArgumentException("Unsupported type: " + primitive);
- }
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java
deleted file mode 100644
index 65b9d44ad4..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReader.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.util.List;
-import java.util.Map;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.orc.OrcRowReader;
-import org.apache.iceberg.orc.OrcSchemaWithTypeVisitor;
-import org.apache.iceberg.orc.OrcValueReader;
-import org.apache.iceberg.orc.OrcValueReaders;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.storage.ql.exec.vector.StructColumnVector;
-import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
-
-public class FlinkOrcReader implements OrcRowReader<RowData> {
- private final OrcValueReader<?> reader;
-
- public FlinkOrcReader(Schema iSchema, TypeDescription readSchema) {
- this(iSchema, readSchema, ImmutableMap.of());
- }
-
- public FlinkOrcReader(Schema iSchema, TypeDescription readSchema, Map<Integer, ?> idToConstant) {
- this.reader =
- OrcSchemaWithTypeVisitor.visit(iSchema, readSchema, new ReadBuilder(idToConstant));
- }
-
- @Override
- public RowData read(VectorizedRowBatch batch, int row) {
- return (RowData) reader.read(new StructColumnVector(batch.size, batch.cols), row);
- }
-
- @Override
- public void setBatchContext(long batchOffsetInFile) {
- reader.setBatchContext(batchOffsetInFile);
- }
-
- private static class ReadBuilder extends OrcSchemaWithTypeVisitor<OrcValueReader<?>> {
- private final Map<Integer, ?> idToConstant;
-
- private ReadBuilder(Map<Integer, ?> idToConstant) {
- this.idToConstant = idToConstant;
- }
-
- @Override
- public OrcValueReader<RowData> record(
- Types.StructType iStruct,
- TypeDescription record,
- List<String> names,
- List<OrcValueReader<?>> fields) {
- return FlinkOrcReaders.struct(fields, iStruct, idToConstant);
- }
-
- @Override
- public OrcValueReader<ArrayData> list(
- Types.ListType iList, TypeDescription array, OrcValueReader<?> elementReader) {
- return FlinkOrcReaders.array(elementReader);
- }
-
- @Override
- public OrcValueReader<MapData> map(
- Types.MapType iMap,
- TypeDescription map,
- OrcValueReader<?> keyReader,
- OrcValueReader<?> valueReader) {
- return FlinkOrcReaders.map(keyReader, valueReader);
- }
-
- @Override
- public OrcValueReader<?> primitive(Type.PrimitiveType iPrimitive, TypeDescription primitive) {
- switch (iPrimitive.typeId()) {
- case BOOLEAN:
- return OrcValueReaders.booleans();
- case INTEGER:
- return OrcValueReaders.ints();
- case LONG:
- return OrcValueReaders.longs();
- case FLOAT:
- return OrcValueReaders.floats();
- case DOUBLE:
- return OrcValueReaders.doubles();
- case DATE:
- return FlinkOrcReaders.dates();
- case TIME:
- return FlinkOrcReaders.times();
- case TIMESTAMP:
- Types.TimestampType timestampType = (Types.TimestampType) iPrimitive;
- if (timestampType.shouldAdjustToUTC()) {
- return FlinkOrcReaders.timestampTzs();
- } else {
- return FlinkOrcReaders.timestamps();
- }
- case STRING:
- return FlinkOrcReaders.strings();
- case UUID:
- case FIXED:
- case BINARY:
- return OrcValueReaders.bytes();
- case DECIMAL:
- Types.DecimalType decimalType = (Types.DecimalType) iPrimitive;
- return FlinkOrcReaders.decimals(decimalType.precision(), decimalType.scale());
- default:
- throw new IllegalArgumentException(
- String.format(
- "Invalid iceberg type %s corresponding to ORC type %s", iPrimitive, primitive));
- }
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java
deleted file mode 100644
index 7a4a15c7e6..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcReaders.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.math.BigDecimal;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
-import java.util.List;
-import java.util.Map;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.data.GenericMapData;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.iceberg.orc.OrcValueReader;
-import org.apache.iceberg.orc.OrcValueReaders;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.types.Types;
-import org.apache.orc.storage.ql.exec.vector.BytesColumnVector;
-import org.apache.orc.storage.ql.exec.vector.ColumnVector;
-import org.apache.orc.storage.ql.exec.vector.DecimalColumnVector;
-import org.apache.orc.storage.ql.exec.vector.ListColumnVector;
-import org.apache.orc.storage.ql.exec.vector.LongColumnVector;
-import org.apache.orc.storage.ql.exec.vector.MapColumnVector;
-import org.apache.orc.storage.ql.exec.vector.TimestampColumnVector;
-import org.apache.orc.storage.serde2.io.HiveDecimalWritable;
-
-class FlinkOrcReaders {
- private FlinkOrcReaders() {}
-
- static OrcValueReader<StringData> strings() {
- return StringReader.INSTANCE;
- }
-
- static OrcValueReader<Integer> dates() {
- return DateReader.INSTANCE;
- }
-
- static OrcValueReader<DecimalData> decimals(int precision, int scale) {
- if (precision <= 18) {
- return new Decimal18Reader(precision, scale);
- } else if (precision <= 38) {
- return new Decimal38Reader(precision, scale);
- } else {
- throw new IllegalArgumentException("Invalid precision: " + precision);
- }
- }
-
- static OrcValueReader<Integer> times() {
- return TimeReader.INSTANCE;
- }
-
- static OrcValueReader<TimestampData> timestamps() {
- return TimestampReader.INSTANCE;
- }
-
- static OrcValueReader<TimestampData> timestampTzs() {
- return TimestampTzReader.INSTANCE;
- }
-
- static <T> OrcValueReader<ArrayData> array(OrcValueReader<T> elementReader) {
- return new ArrayReader<>(elementReader);
- }
-
- public static <K, V> OrcValueReader<MapData> map(
- OrcValueReader<K> keyReader, OrcValueReader<V> valueReader) {
- return new MapReader<>(keyReader, valueReader);
- }
-
- public static OrcValueReader<RowData> struct(
- List<OrcValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
- return new StructReader(readers, struct, idToConstant);
- }
-
- private static class StringReader implements OrcValueReader<StringData> {
- private static final StringReader INSTANCE = new StringReader();
-
- @Override
- public StringData nonNullRead(ColumnVector vector, int row) {
- BytesColumnVector bytesVector = (BytesColumnVector) vector;
- return StringData.fromBytes(
- bytesVector.vector[row], bytesVector.start[row], bytesVector.length[row]);
- }
- }
-
- private static class DateReader implements OrcValueReader<Integer> {
- private static final DateReader INSTANCE = new DateReader();
-
- @Override
- public Integer nonNullRead(ColumnVector vector, int row) {
- return (int) ((LongColumnVector) vector).vector[row];
- }
- }
-
- private static class Decimal18Reader implements OrcValueReader<DecimalData> {
- private final int precision;
- private final int scale;
-
- Decimal18Reader(int precision, int scale) {
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public DecimalData nonNullRead(ColumnVector vector, int row) {
- HiveDecimalWritable value = ((DecimalColumnVector) vector).vector[row];
-
- // The hive ORC writer may will adjust the scale of decimal data.
- Preconditions.checkArgument(
- value.precision() <= precision,
- "Cannot read value as decimal(%s,%s), too large: %s",
- precision,
- scale,
- value);
-
- return DecimalData.fromUnscaledLong(value.serialize64(scale), precision, scale);
- }
- }
-
- private static class Decimal38Reader implements OrcValueReader<DecimalData> {
- private final int precision;
- private final int scale;
-
- Decimal38Reader(int precision, int scale) {
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public DecimalData nonNullRead(ColumnVector vector, int row) {
- BigDecimal value =
- ((DecimalColumnVector) vector).vector[row].getHiveDecimal().bigDecimalValue();
-
- Preconditions.checkArgument(
- value.precision() <= precision,
- "Cannot read value as decimal(%s,%s), too large: %s",
- precision,
- scale,
- value);
-
- return DecimalData.fromBigDecimal(value, precision, scale);
- }
- }
-
- private static class TimeReader implements OrcValueReader<Integer> {
- private static final TimeReader INSTANCE = new TimeReader();
-
- @Override
- public Integer nonNullRead(ColumnVector vector, int row) {
- long micros = ((LongColumnVector) vector).vector[row];
- // Flink only support time mills, just erase micros.
- return (int) (micros / 1000);
- }
- }
-
- private static class TimestampReader implements OrcValueReader<TimestampData> {
- private static final TimestampReader INSTANCE = new TimestampReader();
-
- @Override
- public TimestampData nonNullRead(ColumnVector vector, int row) {
- TimestampColumnVector tcv = (TimestampColumnVector) vector;
- LocalDateTime localDate =
- Instant.ofEpochSecond(Math.floorDiv(tcv.time[row], 1_000), tcv.nanos[row])
- .atOffset(ZoneOffset.UTC)
- .toLocalDateTime();
- return TimestampData.fromLocalDateTime(localDate);
- }
- }
-
- private static class TimestampTzReader implements OrcValueReader<TimestampData> {
- private static final TimestampTzReader INSTANCE = new TimestampTzReader();
-
- @Override
- public TimestampData nonNullRead(ColumnVector vector, int row) {
- TimestampColumnVector tcv = (TimestampColumnVector) vector;
- Instant instant =
- Instant.ofEpochSecond(Math.floorDiv(tcv.time[row], 1_000), tcv.nanos[row])
- .atOffset(ZoneOffset.UTC)
- .toInstant();
- return TimestampData.fromInstant(instant);
- }
- }
-
- private static class ArrayReader<T> implements OrcValueReader<ArrayData> {
- private final OrcValueReader<T> elementReader;
-
- private ArrayReader(OrcValueReader<T> elementReader) {
- this.elementReader = elementReader;
- }
-
- @Override
- public ArrayData nonNullRead(ColumnVector vector, int row) {
- ListColumnVector listVector = (ListColumnVector) vector;
- int offset = (int) listVector.offsets[row];
- int length = (int) listVector.lengths[row];
- List<T> elements = Lists.newArrayListWithExpectedSize(length);
- for (int c = 0; c < length; ++c) {
- elements.add(elementReader.read(listVector.child, offset + c));
- }
- return new GenericArrayData(elements.toArray());
- }
-
- @Override
- public void setBatchContext(long batchOffsetInFile) {
- elementReader.setBatchContext(batchOffsetInFile);
- }
- }
-
- private static class MapReader<K, V> implements OrcValueReader<MapData> {
- private final OrcValueReader<K> keyReader;
- private final OrcValueReader<V> valueReader;
-
- private MapReader(OrcValueReader<K> keyReader, OrcValueReader<V> valueReader) {
- this.keyReader = keyReader;
- this.valueReader = valueReader;
- }
-
- @Override
- public MapData nonNullRead(ColumnVector vector, int row) {
- MapColumnVector mapVector = (MapColumnVector) vector;
- int offset = (int) mapVector.offsets[row];
- long length = mapVector.lengths[row];
-
- Map<K, V> map = Maps.newHashMap();
- for (int c = 0; c < length; c++) {
- K key = keyReader.read(mapVector.keys, offset + c);
- V value = valueReader.read(mapVector.values, offset + c);
- map.put(key, value);
- }
-
- return new GenericMapData(map);
- }
-
- @Override
- public void setBatchContext(long batchOffsetInFile) {
- keyReader.setBatchContext(batchOffsetInFile);
- valueReader.setBatchContext(batchOffsetInFile);
- }
- }
-
- private static class StructReader extends OrcValueReaders.StructReader<RowData> {
- private final int numFields;
-
- StructReader(
- List<OrcValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
- super(readers, struct, idToConstant);
- this.numFields = struct.fields().size();
- }
-
- @Override
- protected RowData create() {
- return new GenericRowData(numFields);
- }
-
- @Override
- protected void set(RowData struct, int pos, Object value) {
- ((GenericRowData) struct).setField(pos, value);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java
deleted file mode 100644
index 6a31accffd..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriter.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.util.Deque;
-import java.util.List;
-import java.util.stream.Stream;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.iceberg.FieldMetrics;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.data.orc.GenericOrcWriters;
-import org.apache.iceberg.orc.OrcRowWriter;
-import org.apache.iceberg.orc.OrcValueWriter;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
-
-public class FlinkOrcWriter implements OrcRowWriter<RowData> {
- private final FlinkOrcWriters.RowDataWriter writer;
-
- private FlinkOrcWriter(RowType rowType, Schema iSchema) {
- this.writer =
- (FlinkOrcWriters.RowDataWriter)
- FlinkSchemaVisitor.visit(rowType, iSchema, new WriteBuilder());
- }
-
- public static OrcRowWriter<RowData> buildWriter(RowType rowType, Schema iSchema) {
- return new FlinkOrcWriter(rowType, iSchema);
- }
-
- @Override
- public void write(RowData row, VectorizedRowBatch output) {
- Preconditions.checkArgument(row != null, "value must not be null");
- writer.writeRow(row, output);
- }
-
- @Override
- public List<OrcValueWriter<?>> writers() {
- return writer.writers();
- }
-
- @Override
- public Stream<FieldMetrics<?>> metrics() {
- return writer.metrics();
- }
-
- private static class WriteBuilder extends FlinkSchemaVisitor<OrcValueWriter<?>> {
- private final Deque<Integer> fieldIds = Lists.newLinkedList();
-
- private WriteBuilder() {}
-
- @Override
- public void beforeField(Types.NestedField field) {
- fieldIds.push(field.fieldId());
- }
-
- @Override
- public void afterField(Types.NestedField field) {
- fieldIds.pop();
- }
-
- @Override
- public OrcValueWriter<RowData> record(
- Types.StructType iStruct, List<OrcValueWriter<?>> results, List<LogicalType> fieldType) {
- return FlinkOrcWriters.struct(results, fieldType);
- }
-
- @Override
- public OrcValueWriter<?> map(
- Types.MapType iMap,
- OrcValueWriter<?> key,
- OrcValueWriter<?> value,
- LogicalType keyType,
- LogicalType valueType) {
- return FlinkOrcWriters.map(key, value, keyType, valueType);
- }
-
- @Override
- public OrcValueWriter<?> list(
- Types.ListType iList, OrcValueWriter<?> element, LogicalType elementType) {
- return FlinkOrcWriters.list(element, elementType);
- }
-
- @Override
- public OrcValueWriter<?> primitive(Type.PrimitiveType iPrimitive, LogicalType flinkPrimitive) {
- switch (iPrimitive.typeId()) {
- case BOOLEAN:
- return GenericOrcWriters.booleans();
- case INTEGER:
- switch (flinkPrimitive.getTypeRoot()) {
- case TINYINT:
- return GenericOrcWriters.bytes();
- case SMALLINT:
- return GenericOrcWriters.shorts();
- }
- return GenericOrcWriters.ints();
- case LONG:
- return GenericOrcWriters.longs();
- case FLOAT:
- Preconditions.checkArgument(
- fieldIds.peek() != null,
- String.format(
- "[BUG] Cannot find field id for primitive field with type %s. This is likely because id "
- + "information is not properly pushed during schema visiting.",
- iPrimitive));
- return GenericOrcWriters.floats(fieldIds.peek());
- case DOUBLE:
- Preconditions.checkArgument(
- fieldIds.peek() != null,
- String.format(
- "[BUG] Cannot find field id for primitive field with type %s. This is likely because id "
- + "information is not properly pushed during schema visiting.",
- iPrimitive));
- return GenericOrcWriters.doubles(fieldIds.peek());
- case DATE:
- return FlinkOrcWriters.dates();
- case TIME:
- return FlinkOrcWriters.times();
- case TIMESTAMP:
- Types.TimestampType timestampType = (Types.TimestampType) iPrimitive;
- if (timestampType.shouldAdjustToUTC()) {
- return FlinkOrcWriters.timestampTzs();
- } else {
- return FlinkOrcWriters.timestamps();
- }
- case STRING:
- return FlinkOrcWriters.strings();
- case UUID:
- case FIXED:
- case BINARY:
- return GenericOrcWriters.byteArrays();
- case DECIMAL:
- Types.DecimalType decimalType = (Types.DecimalType) iPrimitive;
- return FlinkOrcWriters.decimals(decimalType.precision(), decimalType.scale());
- default:
- throw new IllegalArgumentException(
- String.format(
- "Invalid iceberg type %s corresponding to Flink logical type %s",
- iPrimitive, flinkPrimitive));
- }
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java
deleted file mode 100644
index da2f95cf82..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkOrcWriters.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.time.Instant;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
-import java.util.List;
-import java.util.stream.Stream;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.iceberg.FieldMetrics;
-import org.apache.iceberg.data.orc.GenericOrcWriters;
-import org.apache.iceberg.orc.OrcValueWriter;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.orc.storage.common.type.HiveDecimal;
-import org.apache.orc.storage.ql.exec.vector.BytesColumnVector;
-import org.apache.orc.storage.ql.exec.vector.ColumnVector;
-import org.apache.orc.storage.ql.exec.vector.DecimalColumnVector;
-import org.apache.orc.storage.ql.exec.vector.ListColumnVector;
-import org.apache.orc.storage.ql.exec.vector.LongColumnVector;
-import org.apache.orc.storage.ql.exec.vector.MapColumnVector;
-import org.apache.orc.storage.ql.exec.vector.TimestampColumnVector;
-
-class FlinkOrcWriters {
-
- private FlinkOrcWriters() {}
-
- static OrcValueWriter<StringData> strings() {
- return StringWriter.INSTANCE;
- }
-
- static OrcValueWriter<Integer> dates() {
- return DateWriter.INSTANCE;
- }
-
- static OrcValueWriter<Integer> times() {
- return TimeWriter.INSTANCE;
- }
-
- static OrcValueWriter<TimestampData> timestamps() {
- return TimestampWriter.INSTANCE;
- }
-
- static OrcValueWriter<TimestampData> timestampTzs() {
- return TimestampTzWriter.INSTANCE;
- }
-
- static OrcValueWriter<DecimalData> decimals(int precision, int scale) {
- if (precision <= 18) {
- return new Decimal18Writer(precision, scale);
- } else if (precision <= 38) {
- return new Decimal38Writer(precision, scale);
- } else {
- throw new IllegalArgumentException("Invalid precision: " + precision);
- }
- }
-
- static <T> OrcValueWriter<ArrayData> list(
- OrcValueWriter<T> elementWriter, LogicalType elementType) {
- return new ListWriter<>(elementWriter, elementType);
- }
-
- static <K, V> OrcValueWriter<MapData> map(
- OrcValueWriter<K> keyWriter,
- OrcValueWriter<V> valueWriter,
- LogicalType keyType,
- LogicalType valueType) {
- return new MapWriter<>(keyWriter, valueWriter, keyType, valueType);
- }
-
- static OrcValueWriter<RowData> struct(List<OrcValueWriter<?>> writers, List<LogicalType> types) {
- return new RowDataWriter(writers, types);
- }
-
- private static class StringWriter implements OrcValueWriter<StringData> {
- private static final StringWriter INSTANCE = new StringWriter();
-
- @Override
- public void nonNullWrite(int rowId, StringData data, ColumnVector output) {
- byte[] value = data.toBytes();
- ((BytesColumnVector) output).setRef(rowId, value, 0, value.length);
- }
- }
-
- private static class DateWriter implements OrcValueWriter<Integer> {
- private static final DateWriter INSTANCE = new DateWriter();
-
- @Override
- public void nonNullWrite(int rowId, Integer data, ColumnVector output) {
- ((LongColumnVector) output).vector[rowId] = data;
- }
- }
-
- private static class TimeWriter implements OrcValueWriter<Integer> {
- private static final TimeWriter INSTANCE = new TimeWriter();
-
- @Override
- public void nonNullWrite(int rowId, Integer millis, ColumnVector output) {
- // The time in flink is in millisecond, while the standard time in iceberg is microsecond.
- // So we need to transform it to microsecond.
- ((LongColumnVector) output).vector[rowId] = millis * 1000L;
- }
- }
-
- private static class TimestampWriter implements OrcValueWriter<TimestampData> {
- private static final TimestampWriter INSTANCE = new TimestampWriter();
-
- @Override
- public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
- TimestampColumnVector cv = (TimestampColumnVector) output;
- cv.setIsUTC(true);
- // millis
- OffsetDateTime offsetDateTime = data.toInstant().atOffset(ZoneOffset.UTC);
- cv.time[rowId] =
- offsetDateTime.toEpochSecond() * 1_000 + offsetDateTime.getNano() / 1_000_000;
- // truncate nanos to only keep microsecond precision.
- cv.nanos[rowId] = (offsetDateTime.getNano() / 1_000) * 1_000;
- }
- }
-
- private static class TimestampTzWriter implements OrcValueWriter<TimestampData> {
- private static final TimestampTzWriter INSTANCE = new TimestampTzWriter();
-
- @SuppressWarnings("JavaInstantGetSecondsGetNano")
- @Override
- public void nonNullWrite(int rowId, TimestampData data, ColumnVector output) {
- TimestampColumnVector cv = (TimestampColumnVector) output;
- // millis
- Instant instant = data.toInstant();
- cv.time[rowId] = instant.toEpochMilli();
- // truncate nanos to only keep microsecond precision.
- cv.nanos[rowId] = (instant.getNano() / 1_000) * 1_000;
- }
- }
-
- private static class Decimal18Writer implements OrcValueWriter<DecimalData> {
- private final int precision;
- private final int scale;
-
- Decimal18Writer(int precision, int scale) {
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public void nonNullWrite(int rowId, DecimalData data, ColumnVector output) {
- Preconditions.checkArgument(
- scale == data.scale(),
- "Cannot write value as decimal(%s,%s), wrong scale: %s",
- precision,
- scale,
- data);
- Preconditions.checkArgument(
- data.precision() <= precision,
- "Cannot write value as decimal(%s,%s), too large: %s",
- precision,
- scale,
- data);
-
- ((DecimalColumnVector) output)
- .vector[rowId].setFromLongAndScale(data.toUnscaledLong(), data.scale());
- }
- }
-
- private static class Decimal38Writer implements OrcValueWriter<DecimalData> {
- private final int precision;
- private final int scale;
-
- Decimal38Writer(int precision, int scale) {
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public void nonNullWrite(int rowId, DecimalData data, ColumnVector output) {
- Preconditions.checkArgument(
- scale == data.scale(),
- "Cannot write value as decimal(%s,%s), wrong scale: %s",
- precision,
- scale,
- data);
- Preconditions.checkArgument(
- data.precision() <= precision,
- "Cannot write value as decimal(%s,%s), too large: %s",
- precision,
- scale,
- data);
-
- ((DecimalColumnVector) output)
- .vector[rowId].set(HiveDecimal.create(data.toBigDecimal(), false));
- }
- }
-
- static class ListWriter<T> implements OrcValueWriter<ArrayData> {
- private final OrcValueWriter<T> elementWriter;
- private final ArrayData.ElementGetter elementGetter;
-
- ListWriter(OrcValueWriter<T> elementWriter, LogicalType elementType) {
- this.elementWriter = elementWriter;
- this.elementGetter = ArrayData.createElementGetter(elementType);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void nonNullWrite(int rowId, ArrayData data, ColumnVector output) {
- ListColumnVector cv = (ListColumnVector) output;
- cv.lengths[rowId] = data.size();
- cv.offsets[rowId] = cv.childCount;
- cv.childCount = (int) (cv.childCount + cv.lengths[rowId]);
- // make sure the child is big enough.
- growColumnVector(cv.child, cv.childCount);
-
- for (int e = 0; e < cv.lengths[rowId]; ++e) {
- Object value = elementGetter.getElementOrNull(data, e);
- elementWriter.write((int) (e + cv.offsets[rowId]), (T) value, cv.child);
- }
- }
-
- @Override
- public Stream<FieldMetrics<?>> metrics() {
- return elementWriter.metrics();
- }
- }
-
- static class MapWriter<K, V> implements OrcValueWriter<MapData> {
- private final OrcValueWriter<K> keyWriter;
- private final OrcValueWriter<V> valueWriter;
- private final ArrayData.ElementGetter keyGetter;
- private final ArrayData.ElementGetter valueGetter;
-
- MapWriter(
- OrcValueWriter<K> keyWriter,
- OrcValueWriter<V> valueWriter,
- LogicalType keyType,
- LogicalType valueType) {
- this.keyWriter = keyWriter;
- this.valueWriter = valueWriter;
- this.keyGetter = ArrayData.createElementGetter(keyType);
- this.valueGetter = ArrayData.createElementGetter(valueType);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void nonNullWrite(int rowId, MapData data, ColumnVector output) {
- MapColumnVector cv = (MapColumnVector) output;
- ArrayData keyArray = data.keyArray();
- ArrayData valArray = data.valueArray();
-
- // record the length and start of the list elements
- cv.lengths[rowId] = data.size();
- cv.offsets[rowId] = cv.childCount;
- cv.childCount = (int) (cv.childCount + cv.lengths[rowId]);
- // make sure the child is big enough
- growColumnVector(cv.keys, cv.childCount);
- growColumnVector(cv.values, cv.childCount);
- // Add each element
- for (int e = 0; e < cv.lengths[rowId]; ++e) {
- int pos = (int) (e + cv.offsets[rowId]);
- keyWriter.write(pos, (K) keyGetter.getElementOrNull(keyArray, e), cv.keys);
- valueWriter.write(pos, (V) valueGetter.getElementOrNull(valArray, e), cv.values);
- }
- }
-
- @Override
- public Stream<FieldMetrics<?>> metrics() {
- return Stream.concat(keyWriter.metrics(), valueWriter.metrics());
- }
- }
-
- static class RowDataWriter extends GenericOrcWriters.StructWriter<RowData> {
- private final List<RowData.FieldGetter> fieldGetters;
-
- RowDataWriter(List<OrcValueWriter<?>> writers, List<LogicalType> types) {
- super(writers);
-
- this.fieldGetters = Lists.newArrayListWithExpectedSize(types.size());
- for (int i = 0; i < types.size(); i++) {
- fieldGetters.add(RowData.createFieldGetter(types.get(i), i));
- }
- }
-
- @Override
- protected Object get(RowData struct, int index) {
- return fieldGetters.get(index).getFieldOrNull(struct);
- }
- }
-
- private static void growColumnVector(ColumnVector cv, int requestedSize) {
- if (cv.isNull.length < requestedSize) {
- // Use growth factor of 3 to avoid frequent array allocations
- cv.ensureSize(requestedSize * 3, true);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java
deleted file mode 100644
index ab7b1174c9..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java
+++ /dev/null
@@ -1,832 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RawValueData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.iceberg.MetadataColumns;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.parquet.ParquetSchemaUtil;
-import org.apache.iceberg.parquet.ParquetValueReader;
-import org.apache.iceberg.parquet.ParquetValueReaders;
-import org.apache.iceberg.parquet.TypeWithSchemaVisitor;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.types.Types;
-import org.apache.iceberg.util.ArrayUtil;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.io.api.Binary;
-import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-
-public class FlinkParquetReaders {
- private FlinkParquetReaders() {}
-
- public static ParquetValueReader<RowData> buildReader(
- Schema expectedSchema, MessageType fileSchema) {
- return buildReader(expectedSchema, fileSchema, ImmutableMap.of());
- }
-
- @SuppressWarnings("unchecked")
- public static ParquetValueReader<RowData> buildReader(
- Schema expectedSchema, MessageType fileSchema, Map<Integer, ?> idToConstant) {
- return (ParquetValueReader<RowData>)
- TypeWithSchemaVisitor.visit(
- expectedSchema.asStruct(), fileSchema, new ReadBuilder(fileSchema, idToConstant));
- }
-
- private static class ReadBuilder extends TypeWithSchemaVisitor<ParquetValueReader<?>> {
- private final MessageType type;
- private final Map<Integer, ?> idToConstant;
-
- ReadBuilder(MessageType type, Map<Integer, ?> idToConstant) {
- this.type = type;
- this.idToConstant = idToConstant;
- }
-
- @Override
- public ParquetValueReader<RowData> message(
- Types.StructType expected, MessageType message, List<ParquetValueReader<?>> fieldReaders) {
- return struct(expected, message.asGroupType(), fieldReaders);
- }
-
- @Override
- public ParquetValueReader<RowData> struct(
- Types.StructType expected, GroupType struct, List<ParquetValueReader<?>> fieldReaders) {
- // match the expected struct's order
- Map<Integer, ParquetValueReader<?>> readersById = Maps.newHashMap();
- Map<Integer, Type> typesById = Maps.newHashMap();
- Map<Integer, Integer> maxDefinitionLevelsById = Maps.newHashMap();
- List<Type> fields = struct.getFields();
- for (int i = 0; i < fields.size(); i += 1) {
- Type fieldType = fields.get(i);
- if (fieldReaders.get(i) != null) {
- int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1;
- if (fieldType.getId() != null) {
- int id = fieldType.getId().intValue();
- readersById.put(id, ParquetValueReaders.option(fieldType, fieldD, fieldReaders.get(i)));
- typesById.put(id, fieldType);
- if (idToConstant.containsKey(id)) {
- maxDefinitionLevelsById.put(id, fieldD);
- }
- }
- }
- }
-
- List<Types.NestedField> expectedFields =
- expected != null ? expected.fields() : ImmutableList.of();
- List<ParquetValueReader<?>> reorderedFields =
- Lists.newArrayListWithExpectedSize(expectedFields.size());
- List<Type> types = Lists.newArrayListWithExpectedSize(expectedFields.size());
- // Defaulting to parent max definition level
- int defaultMaxDefinitionLevel = type.getMaxDefinitionLevel(currentPath());
- for (Types.NestedField field : expectedFields) {
- int id = field.fieldId();
- if (idToConstant.containsKey(id)) {
- // containsKey is used because the constant may be null
- int fieldMaxDefinitionLevel =
- maxDefinitionLevelsById.getOrDefault(id, defaultMaxDefinitionLevel);
- reorderedFields.add(
- ParquetValueReaders.constant(idToConstant.get(id), fieldMaxDefinitionLevel));
- types.add(null);
- } else if (id == MetadataColumns.ROW_POSITION.fieldId()) {
- reorderedFields.add(ParquetValueReaders.position());
- types.add(null);
- } else if (id == MetadataColumns.IS_DELETED.fieldId()) {
- reorderedFields.add(ParquetValueReaders.constant(false));
- types.add(null);
- } else {
- ParquetValueReader<?> reader = readersById.get(id);
- if (reader != null) {
- reorderedFields.add(reader);
- types.add(typesById.get(id));
- } else {
- reorderedFields.add(ParquetValueReaders.nulls());
- types.add(null);
- }
- }
- }
-
- return new RowDataReader(types, reorderedFields);
- }
-
- @Override
- public ParquetValueReader<?> list(
- Types.ListType expectedList, GroupType array, ParquetValueReader<?> elementReader) {
- if (expectedList == null) {
- return null;
- }
-
- String[] repeatedPath = currentPath();
-
- int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
- int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;
-
- Type elementType = ParquetSchemaUtil.determineListElementType(array);
- int elementD = type.getMaxDefinitionLevel(path(elementType.getName())) - 1;
-
- return new ArrayReader<>(
- repeatedD, repeatedR, ParquetValueReaders.option(elementType, elementD, elementReader));
- }
-
- @Override
- public ParquetValueReader<?> map(
- Types.MapType expectedMap,
- GroupType map,
- ParquetValueReader<?> keyReader,
- ParquetValueReader<?> valueReader) {
- if (expectedMap == null) {
- return null;
- }
-
- GroupType repeatedKeyValue = map.getFields().get(0).asGroupType();
- String[] repeatedPath = currentPath();
-
- int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
- int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;
-
- Type keyType = repeatedKeyValue.getType(0);
- int keyD = type.getMaxDefinitionLevel(path(keyType.getName())) - 1;
- Type valueType = repeatedKeyValue.getType(1);
- int valueD = type.getMaxDefinitionLevel(path(valueType.getName())) - 1;
-
- return new MapReader<>(
- repeatedD,
- repeatedR,
- ParquetValueReaders.option(keyType, keyD, keyReader),
- ParquetValueReaders.option(valueType, valueD, valueReader));
- }
-
- @Override
- @SuppressWarnings("CyclomaticComplexity")
- public ParquetValueReader<?> primitive(
- org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) {
- if (expected == null) {
- return null;
- }
-
- ColumnDescriptor desc = type.getColumnDescription(currentPath());
-
- if (primitive.getOriginalType() != null) {
- switch (primitive.getOriginalType()) {
- case ENUM:
- case JSON:
- case UTF8:
- return new StringReader(desc);
- case INT_8:
- case INT_16:
- case INT_32:
- if (expected.typeId() == Types.LongType.get().typeId()) {
- return new ParquetValueReaders.IntAsLongReader(desc);
- } else {
- return new ParquetValueReaders.UnboxedReader<>(desc);
- }
- case TIME_MICROS:
- return new LossyMicrosToMillisTimeReader(desc);
- case TIME_MILLIS:
- return new MillisTimeReader(desc);
- case DATE:
- case INT_64:
- return new ParquetValueReaders.UnboxedReader<>(desc);
- case TIMESTAMP_MICROS:
- if (((Types.TimestampType) expected).shouldAdjustToUTC()) {
- return new MicrosToTimestampTzReader(desc);
- } else {
- return new MicrosToTimestampReader(desc);
- }
- case TIMESTAMP_MILLIS:
- if (((Types.TimestampType) expected).shouldAdjustToUTC()) {
- return new MillisToTimestampTzReader(desc);
- } else {
- return new MillisToTimestampReader(desc);
- }
- case DECIMAL:
- DecimalLogicalTypeAnnotation decimal =
- (DecimalLogicalTypeAnnotation) primitive.getLogicalTypeAnnotation();
- switch (primitive.getPrimitiveTypeName()) {
- case BINARY:
- case FIXED_LEN_BYTE_ARRAY:
- return new BinaryDecimalReader(desc, decimal.getPrecision(), decimal.getScale());
- case INT64:
- return new LongDecimalReader(desc, decimal.getPrecision(), decimal.getScale());
- case INT32:
- return new IntegerDecimalReader(desc, decimal.getPrecision(), decimal.getScale());
- default:
- throw new UnsupportedOperationException(
- "Unsupported base type for decimal: " + primitive.getPrimitiveTypeName());
- }
- case BSON:
- return new ParquetValueReaders.ByteArrayReader(desc);
- default:
- throw new UnsupportedOperationException(
- "Unsupported logical type: " + primitive.getOriginalType());
- }
- }
-
- switch (primitive.getPrimitiveTypeName()) {
- case FIXED_LEN_BYTE_ARRAY:
- case BINARY:
- return new ParquetValueReaders.ByteArrayReader(desc);
- case INT32:
- if (expected.typeId() == org.apache.iceberg.types.Type.TypeID.LONG) {
- return new ParquetValueReaders.IntAsLongReader(desc);
- } else {
- return new ParquetValueReaders.UnboxedReader<>(desc);
- }
- case FLOAT:
- if (expected.typeId() == org.apache.iceberg.types.Type.TypeID.DOUBLE) {
- return new ParquetValueReaders.FloatAsDoubleReader(desc);
- } else {
- return new ParquetValueReaders.UnboxedReader<>(desc);
- }
- case BOOLEAN:
- case INT64:
- case DOUBLE:
- return new ParquetValueReaders.UnboxedReader<>(desc);
- default:
- throw new UnsupportedOperationException("Unsupported type: " + primitive);
- }
- }
- }
-
- private static class BinaryDecimalReader
- extends ParquetValueReaders.PrimitiveReader<DecimalData> {
- private final int precision;
- private final int scale;
-
- BinaryDecimalReader(ColumnDescriptor desc, int precision, int scale) {
- super(desc);
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public DecimalData read(DecimalData ignored) {
- Binary binary = column.nextBinary();
- BigDecimal bigDecimal = new BigDecimal(new BigInteger(binary.getBytes()), scale);
- // TODO: need a unit test to write-read-validate decimal via FlinkParquetWrite/Reader
- return DecimalData.fromBigDecimal(bigDecimal, precision, scale);
- }
- }
-
- private static class IntegerDecimalReader
- extends ParquetValueReaders.PrimitiveReader<DecimalData> {
- private final int precision;
- private final int scale;
-
- IntegerDecimalReader(ColumnDescriptor desc, int precision, int scale) {
- super(desc);
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public DecimalData read(DecimalData ignored) {
- return DecimalData.fromUnscaledLong(column.nextInteger(), precision, scale);
- }
- }
-
- private static class LongDecimalReader extends ParquetValueReaders.PrimitiveReader<DecimalData> {
- private final int precision;
- private final int scale;
-
- LongDecimalReader(ColumnDescriptor desc, int precision, int scale) {
- super(desc);
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public DecimalData read(DecimalData ignored) {
- return DecimalData.fromUnscaledLong(column.nextLong(), precision, scale);
- }
- }
-
- private static class MicrosToTimestampTzReader
- extends ParquetValueReaders.UnboxedReader<TimestampData> {
- MicrosToTimestampTzReader(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public TimestampData read(TimestampData ignored) {
- long value = readLong();
- return TimestampData.fromLocalDateTime(
- Instant.ofEpochSecond(
- Math.floorDiv(value, 1000_000), Math.floorMod(value, 1000_000) * 1000)
- .atOffset(ZoneOffset.UTC)
- .toLocalDateTime());
- }
-
- @Override
- public long readLong() {
- return column.nextLong();
- }
- }
-
- private static class MicrosToTimestampReader
- extends ParquetValueReaders.UnboxedReader<TimestampData> {
- MicrosToTimestampReader(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public TimestampData read(TimestampData ignored) {
- long value = readLong();
- return TimestampData.fromInstant(
- Instant.ofEpochSecond(
- Math.floorDiv(value, 1000_000), Math.floorMod(value, 1000_000) * 1000));
- }
-
- @Override
- public long readLong() {
- return column.nextLong();
- }
- }
-
- private static class MillisToTimestampReader
- extends ParquetValueReaders.UnboxedReader<TimestampData> {
- MillisToTimestampReader(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public TimestampData read(TimestampData ignored) {
- long millis = readLong();
- return TimestampData.fromEpochMillis(millis);
- }
-
- @Override
- public long readLong() {
- return column.nextLong();
- }
- }
-
- private static class MillisToTimestampTzReader
- extends ParquetValueReaders.UnboxedReader<TimestampData> {
- MillisToTimestampTzReader(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public TimestampData read(TimestampData ignored) {
- long millis = readLong();
- return TimestampData.fromLocalDateTime(
- Instant.ofEpochMilli(millis).atOffset(ZoneOffset.UTC).toLocalDateTime());
- }
-
- @Override
- public long readLong() {
- return column.nextLong();
- }
- }
-
- private static class StringReader extends ParquetValueReaders.PrimitiveReader<StringData> {
- StringReader(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public StringData read(StringData ignored) {
- Binary binary = column.nextBinary();
- ByteBuffer buffer = binary.toByteBuffer();
- if (buffer.hasArray()) {
- return StringData.fromBytes(
- buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
- } else {
- return StringData.fromBytes(binary.getBytes());
- }
- }
- }
-
- private static class LossyMicrosToMillisTimeReader
- extends ParquetValueReaders.PrimitiveReader<Integer> {
- LossyMicrosToMillisTimeReader(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public Integer read(Integer reuse) {
- // Discard microseconds since Flink uses millisecond unit for TIME type.
- return (int) Math.floorDiv(column.nextLong(), 1000);
- }
- }
-
- private static class MillisTimeReader extends ParquetValueReaders.PrimitiveReader<Integer> {
- MillisTimeReader(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public Integer read(Integer reuse) {
- return (int) column.nextLong();
- }
- }
-
- private static class ArrayReader<E>
- extends ParquetValueReaders.RepeatedReader<ArrayData, ReusableArrayData, E> {
- private int readPos = 0;
- private int writePos = 0;
-
- ArrayReader(int definitionLevel, int repetitionLevel, ParquetValueReader<E> reader) {
- super(definitionLevel, repetitionLevel, reader);
- }
-
- @Override
- protected ReusableArrayData newListData(ArrayData reuse) {
- this.readPos = 0;
- this.writePos = 0;
-
- if (reuse instanceof ReusableArrayData) {
- return (ReusableArrayData) reuse;
- } else {
- return new ReusableArrayData();
- }
- }
-
- @Override
- @SuppressWarnings("unchecked")
- protected E getElement(ReusableArrayData list) {
- E value = null;
- if (readPos < list.capacity()) {
- value = (E) list.values[readPos];
- }
-
- readPos += 1;
-
- return value;
- }
-
- @Override
- protected void addElement(ReusableArrayData reused, E element) {
- if (writePos >= reused.capacity()) {
- reused.grow();
- }
-
- reused.values[writePos] = element;
-
- writePos += 1;
- }
-
- @Override
- protected ArrayData buildList(ReusableArrayData list) {
- // Since ReusableArrayData is not accepted by Flink, use GenericArrayData temporarily to walk
- // around it.
- // Revert this to use ReusableArrayData once it is fixed in Flink.
- // For your reference, https://issues.apache.org/jira/browse/FLINK-25238.
- return new GenericArrayData(Arrays.copyOf(list.values, writePos));
- }
- }
-
- private static class MapReader<K, V>
- extends ParquetValueReaders.RepeatedKeyValueReader<MapData, ReusableMapData, K, V> {
- private int readPos = 0;
- private int writePos = 0;
-
- private final ParquetValueReaders.ReusableEntry<K, V> entry =
- new ParquetValueReaders.ReusableEntry<>();
- private final ParquetValueReaders.ReusableEntry<K, V> nullEntry =
- new ParquetValueReaders.ReusableEntry<>();
-
- MapReader(
- int definitionLevel,
- int repetitionLevel,
- ParquetValueReader<K> keyReader,
- ParquetValueReader<V> valueReader) {
- super(definitionLevel, repetitionLevel, keyReader, valueReader);
- }
-
- @Override
- protected ReusableMapData newMapData(MapData reuse) {
- this.readPos = 0;
- this.writePos = 0;
-
- if (reuse instanceof ReusableMapData) {
- return (ReusableMapData) reuse;
- } else {
- return new ReusableMapData();
- }
- }
-
- @Override
- @SuppressWarnings("unchecked")
- protected Map.Entry<K, V> getPair(ReusableMapData map) {
- Map.Entry<K, V> kv = nullEntry;
- if (readPos < map.capacity()) {
- entry.set((K) map.keys.values[readPos], (V) map.values.values[readPos]);
- kv = entry;
- }
-
- readPos += 1;
-
- return kv;
- }
-
- @Override
- protected void addPair(ReusableMapData map, K key, V value) {
- if (writePos >= map.capacity()) {
- map.grow();
- }
-
- map.keys.values[writePos] = key;
- map.values.values[writePos] = value;
-
- writePos += 1;
- }
-
- @Override
- protected MapData buildMap(ReusableMapData map) {
- map.setNumElements(writePos);
- return map;
- }
- }
-
- private static class RowDataReader
- extends ParquetValueReaders.StructReader<RowData, GenericRowData> {
- private final int numFields;
-
- RowDataReader(List<Type> types, List<ParquetValueReader<?>> readers) {
- super(types, readers);
- this.numFields = readers.size();
- }
-
- @Override
- protected GenericRowData newStructData(RowData reuse) {
- if (reuse instanceof GenericRowData) {
- return (GenericRowData) reuse;
- } else {
- return new GenericRowData(numFields);
- }
- }
-
- @Override
- protected Object getField(GenericRowData intermediate, int pos) {
- return intermediate.getField(pos);
- }
-
- @Override
- protected RowData buildStruct(GenericRowData struct) {
- return struct;
- }
-
- @Override
- protected void set(GenericRowData row, int pos, Object value) {
- row.setField(pos, value);
- }
-
- @Override
- protected void setNull(GenericRowData row, int pos) {
- row.setField(pos, null);
- }
-
- @Override
- protected void setBoolean(GenericRowData row, int pos, boolean value) {
- row.setField(pos, value);
- }
-
- @Override
- protected void setInteger(GenericRowData row, int pos, int value) {
- row.setField(pos, value);
- }
-
- @Override
- protected void setLong(GenericRowData row, int pos, long value) {
- row.setField(pos, value);
- }
-
- @Override
- protected void setFloat(GenericRowData row, int pos, float value) {
- row.setField(pos, value);
- }
-
- @Override
- protected void setDouble(GenericRowData row, int pos, double value) {
- row.setField(pos, value);
- }
- }
-
- private static class ReusableMapData implements MapData {
- private final ReusableArrayData keys;
- private final ReusableArrayData values;
-
- private int numElements;
-
- private ReusableMapData() {
- this.keys = new ReusableArrayData();
- this.values = new ReusableArrayData();
- }
-
- private void grow() {
- keys.grow();
- values.grow();
- }
-
- private int capacity() {
- return keys.capacity();
- }
-
- public void setNumElements(int numElements) {
- this.numElements = numElements;
- keys.setNumElements(numElements);
- values.setNumElements(numElements);
- }
-
- @Override
- public int size() {
- return numElements;
- }
-
- @Override
- public ReusableArrayData keyArray() {
- return keys;
- }
-
- @Override
- public ReusableArrayData valueArray() {
- return values;
- }
- }
-
- private static class ReusableArrayData implements ArrayData {
- private static final Object[] EMPTY = new Object[0];
-
- private Object[] values = EMPTY;
- private int numElements = 0;
-
- private void grow() {
- if (values.length == 0) {
- this.values = new Object[20];
- } else {
- Object[] old = values;
- this.values = new Object[old.length << 1];
- // copy the old array in case it has values that can be reused
- System.arraycopy(old, 0, values, 0, old.length);
- }
- }
-
- private int capacity() {
- return values.length;
- }
-
- public void setNumElements(int numElements) {
- this.numElements = numElements;
- }
-
- @Override
- public int size() {
- return numElements;
- }
-
- @Override
- public boolean isNullAt(int ordinal) {
- return null == values[ordinal];
- }
-
- @Override
- public boolean getBoolean(int ordinal) {
- return (boolean) values[ordinal];
- }
-
- @Override
- public byte getByte(int ordinal) {
- return (byte) values[ordinal];
- }
-
- @Override
- public short getShort(int ordinal) {
- return (short) values[ordinal];
- }
-
- @Override
- public int getInt(int ordinal) {
- return (int) values[ordinal];
- }
-
- @Override
- public long getLong(int ordinal) {
- return (long) values[ordinal];
- }
-
- @Override
- public float getFloat(int ordinal) {
- return (float) values[ordinal];
- }
-
- @Override
- public double getDouble(int ordinal) {
- return (double) values[ordinal];
- }
-
- @Override
- public StringData getString(int pos) {
- return (StringData) values[pos];
- }
-
- @Override
- public DecimalData getDecimal(int pos, int precision, int scale) {
- return (DecimalData) values[pos];
- }
-
- @Override
- public TimestampData getTimestamp(int pos, int precision) {
- return (TimestampData) values[pos];
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> RawValueData<T> getRawValue(int pos) {
- return (RawValueData<T>) values[pos];
- }
-
- @Override
- public byte[] getBinary(int ordinal) {
- return (byte[]) values[ordinal];
- }
-
- @Override
- public ArrayData getArray(int ordinal) {
- return (ArrayData) values[ordinal];
- }
-
- @Override
- public MapData getMap(int ordinal) {
- return (MapData) values[ordinal];
- }
-
- @Override
- public RowData getRow(int pos, int numFields) {
- return (RowData) values[pos];
- }
-
- @Override
- public boolean[] toBooleanArray() {
- return ArrayUtil.toPrimitive((Boolean[]) values);
- }
-
- @Override
- public byte[] toByteArray() {
- return ArrayUtil.toPrimitive((Byte[]) values);
- }
-
- @Override
- public short[] toShortArray() {
- return ArrayUtil.toPrimitive((Short[]) values);
- }
-
- @Override
- public int[] toIntArray() {
- return ArrayUtil.toPrimitive((Integer[]) values);
- }
-
- @Override
- public long[] toLongArray() {
- return ArrayUtil.toPrimitive((Long[]) values);
- }
-
- @Override
- public float[] toFloatArray() {
- return ArrayUtil.toPrimitive((Float[]) values);
- }
-
- @Override
- public double[] toDoubleArray() {
- return ArrayUtil.toPrimitive((Double[]) values);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
deleted file mode 100644
index db4f1730a1..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
+++ /dev/null
@@ -1,504 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.MapType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.logical.RowType.RowField;
-import org.apache.flink.table.types.logical.SmallIntType;
-import org.apache.flink.table.types.logical.TinyIntType;
-import org.apache.iceberg.parquet.ParquetValueReaders;
-import org.apache.iceberg.parquet.ParquetValueWriter;
-import org.apache.iceberg.parquet.ParquetValueWriters;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.types.TypeUtil;
-import org.apache.iceberg.util.DecimalUtil;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.io.api.Binary;
-import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-
-public class FlinkParquetWriters {
- private FlinkParquetWriters() {}
-
- @SuppressWarnings("unchecked")
- public static <T> ParquetValueWriter<T> buildWriter(LogicalType schema, MessageType type) {
- return (ParquetValueWriter<T>)
- ParquetWithFlinkSchemaVisitor.visit(schema, type, new WriteBuilder(type));
- }
-
- private static class WriteBuilder extends ParquetWithFlinkSchemaVisitor<ParquetValueWriter<?>> {
- private final MessageType type;
-
- WriteBuilder(MessageType type) {
- this.type = type;
- }
-
- @Override
- public ParquetValueWriter<?> message(
- RowType sStruct, MessageType message, List<ParquetValueWriter<?>> fields) {
- return struct(sStruct, message.asGroupType(), fields);
- }
-
- @Override
- public ParquetValueWriter<?> struct(
- RowType sStruct, GroupType struct, List<ParquetValueWriter<?>> fieldWriters) {
- List<Type> fields = struct.getFields();
- List<RowField> flinkFields = sStruct.getFields();
- List<ParquetValueWriter<?>> writers = Lists.newArrayListWithExpectedSize(fieldWriters.size());
- List<LogicalType> flinkTypes = Lists.newArrayList();
- for (int i = 0; i < fields.size(); i += 1) {
- writers.add(newOption(struct.getType(i), fieldWriters.get(i)));
- flinkTypes.add(flinkFields.get(i).getType());
- }
-
- return new RowDataWriter(writers, flinkTypes);
- }
-
- @Override
- public ParquetValueWriter<?> list(
- ArrayType sArray, GroupType array, ParquetValueWriter<?> elementWriter) {
- GroupType repeated = array.getFields().get(0).asGroupType();
- String[] repeatedPath = currentPath();
-
- int repeatedD = type.getMaxDefinitionLevel(repeatedPath);
- int repeatedR = type.getMaxRepetitionLevel(repeatedPath);
-
- return new ArrayDataWriter<>(
- repeatedD,
- repeatedR,
- newOption(repeated.getType(0), elementWriter),
- sArray.getElementType());
- }
-
- @Override
- public ParquetValueWriter<?> map(
- MapType sMap,
- GroupType map,
- ParquetValueWriter<?> keyWriter,
- ParquetValueWriter<?> valueWriter) {
- GroupType repeatedKeyValue = map.getFields().get(0).asGroupType();
- String[] repeatedPath = currentPath();
-
- int repeatedD = type.getMaxDefinitionLevel(repeatedPath);
- int repeatedR = type.getMaxRepetitionLevel(repeatedPath);
-
- return new MapDataWriter<>(
- repeatedD,
- repeatedR,
- newOption(repeatedKeyValue.getType(0), keyWriter),
- newOption(repeatedKeyValue.getType(1), valueWriter),
- sMap.getKeyType(),
- sMap.getValueType());
- }
-
- private ParquetValueWriter<?> newOption(Type fieldType, ParquetValueWriter<?> writer) {
- int maxD = type.getMaxDefinitionLevel(path(fieldType.getName()));
- return ParquetValueWriters.option(fieldType, maxD, writer);
- }
-
- @Override
- public ParquetValueWriter<?> primitive(LogicalType fType, PrimitiveType primitive) {
- ColumnDescriptor desc = type.getColumnDescription(currentPath());
-
- if (primitive.getOriginalType() != null) {
- switch (primitive.getOriginalType()) {
- case ENUM:
- case JSON:
- case UTF8:
- return strings(desc);
- case DATE:
- case INT_8:
- case INT_16:
- case INT_32:
- return ints(fType, desc);
- case INT_64:
- return ParquetValueWriters.longs(desc);
- case TIME_MICROS:
- return timeMicros(desc);
- case TIMESTAMP_MICROS:
- return timestamps(desc);
- case DECIMAL:
- DecimalLogicalTypeAnnotation decimal =
- (DecimalLogicalTypeAnnotation) primitive.getLogicalTypeAnnotation();
- switch (primitive.getPrimitiveTypeName()) {
- case INT32:
- return decimalAsInteger(desc, decimal.getPrecision(), decimal.getScale());
- case INT64:
- return decimalAsLong(desc, decimal.getPrecision(), decimal.getScale());
- case BINARY:
- case FIXED_LEN_BYTE_ARRAY:
- return decimalAsFixed(desc, decimal.getPrecision(), decimal.getScale());
- default:
- throw new UnsupportedOperationException(
- "Unsupported base type for decimal: " + primitive.getPrimitiveTypeName());
- }
- case BSON:
- return byteArrays(desc);
- default:
- throw new UnsupportedOperationException(
- "Unsupported logical type: " + primitive.getOriginalType());
- }
- }
-
- switch (primitive.getPrimitiveTypeName()) {
- case FIXED_LEN_BYTE_ARRAY:
- case BINARY:
- return byteArrays(desc);
- case BOOLEAN:
- return ParquetValueWriters.booleans(desc);
- case INT32:
- return ints(fType, desc);
- case INT64:
- return ParquetValueWriters.longs(desc);
- case FLOAT:
- return ParquetValueWriters.floats(desc);
- case DOUBLE:
- return ParquetValueWriters.doubles(desc);
- default:
- throw new UnsupportedOperationException("Unsupported type: " + primitive);
- }
- }
- }
-
- private static ParquetValueWriters.PrimitiveWriter<?> ints(
- LogicalType type, ColumnDescriptor desc) {
- if (type instanceof TinyIntType) {
- return ParquetValueWriters.tinyints(desc);
- } else if (type instanceof SmallIntType) {
- return ParquetValueWriters.shorts(desc);
- }
- return ParquetValueWriters.ints(desc);
- }
-
- private static ParquetValueWriters.PrimitiveWriter<StringData> strings(ColumnDescriptor desc) {
- return new StringDataWriter(desc);
- }
-
- private static ParquetValueWriters.PrimitiveWriter<Integer> timeMicros(ColumnDescriptor desc) {
- return new TimeMicrosWriter(desc);
- }
-
- private static ParquetValueWriters.PrimitiveWriter<DecimalData> decimalAsInteger(
- ColumnDescriptor desc, int precision, int scale) {
- Preconditions.checkArgument(
- precision <= 9,
- "Cannot write decimal value as integer with precision larger than 9,"
- + " wrong precision %s",
- precision);
- return new IntegerDecimalWriter(desc, precision, scale);
- }
-
- private static ParquetValueWriters.PrimitiveWriter<DecimalData> decimalAsLong(
- ColumnDescriptor desc, int precision, int scale) {
- Preconditions.checkArgument(
- precision <= 18,
- "Cannot write decimal value as long with precision larger than 18, "
- + " wrong precision %s",
- precision);
- return new LongDecimalWriter(desc, precision, scale);
- }
-
- private static ParquetValueWriters.PrimitiveWriter<DecimalData> decimalAsFixed(
- ColumnDescriptor desc, int precision, int scale) {
- return new FixedDecimalWriter(desc, precision, scale);
- }
-
- private static ParquetValueWriters.PrimitiveWriter<TimestampData> timestamps(
- ColumnDescriptor desc) {
- return new TimestampDataWriter(desc);
- }
-
- private static ParquetValueWriters.PrimitiveWriter<byte[]> byteArrays(ColumnDescriptor desc) {
- return new ByteArrayWriter(desc);
- }
-
- private static class StringDataWriter extends ParquetValueWriters.PrimitiveWriter<StringData> {
- private StringDataWriter(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public void write(int repetitionLevel, StringData value) {
- column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(value.toBytes()));
- }
- }
-
- private static class TimeMicrosWriter extends ParquetValueWriters.PrimitiveWriter<Integer> {
- private TimeMicrosWriter(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public void write(int repetitionLevel, Integer value) {
- long micros = value.longValue() * 1000;
- column.writeLong(repetitionLevel, micros);
- }
- }
-
- private static class IntegerDecimalWriter
- extends ParquetValueWriters.PrimitiveWriter<DecimalData> {
- private final int precision;
- private final int scale;
-
- private IntegerDecimalWriter(ColumnDescriptor desc, int precision, int scale) {
- super(desc);
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public void write(int repetitionLevel, DecimalData decimal) {
- Preconditions.checkArgument(
- decimal.scale() == scale,
- "Cannot write value as decimal(%s,%s), wrong scale: %s",
- precision,
- scale,
- decimal);
- Preconditions.checkArgument(
- decimal.precision() <= precision,
- "Cannot write value as decimal(%s,%s), too large: %s",
- precision,
- scale,
- decimal);
-
- column.writeInteger(repetitionLevel, (int) decimal.toUnscaledLong());
- }
- }
-
- private static class LongDecimalWriter extends ParquetValueWriters.PrimitiveWriter<DecimalData> {
- private final int precision;
- private final int scale;
-
- private LongDecimalWriter(ColumnDescriptor desc, int precision, int scale) {
- super(desc);
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public void write(int repetitionLevel, DecimalData decimal) {
- Preconditions.checkArgument(
- decimal.scale() == scale,
- "Cannot write value as decimal(%s,%s), wrong scale: %s",
- precision,
- scale,
- decimal);
- Preconditions.checkArgument(
- decimal.precision() <= precision,
- "Cannot write value as decimal(%s,%s), too large: %s",
- precision,
- scale,
- decimal);
-
- column.writeLong(repetitionLevel, decimal.toUnscaledLong());
- }
- }
-
- private static class FixedDecimalWriter extends ParquetValueWriters.PrimitiveWriter<DecimalData> {
- private final int precision;
- private final int scale;
- private final ThreadLocal<byte[]> bytes;
-
- private FixedDecimalWriter(ColumnDescriptor desc, int precision, int scale) {
- super(desc);
- this.precision = precision;
- this.scale = scale;
- this.bytes =
- ThreadLocal.withInitial(() -> new byte[TypeUtil.decimalRequiredBytes(precision)]);
- }
-
- @Override
- public void write(int repetitionLevel, DecimalData decimal) {
- byte[] binary =
- DecimalUtil.toReusedFixLengthBytes(precision, scale, decimal.toBigDecimal(), bytes.get());
- column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(binary));
- }
- }
-
- private static class TimestampDataWriter
- extends ParquetValueWriters.PrimitiveWriter<TimestampData> {
- private TimestampDataWriter(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public void write(int repetitionLevel, TimestampData value) {
- column.writeLong(
- repetitionLevel, value.getMillisecond() * 1000 + value.getNanoOfMillisecond() / 1000);
- }
- }
-
- private static class ByteArrayWriter extends ParquetValueWriters.PrimitiveWriter<byte[]> {
- private ByteArrayWriter(ColumnDescriptor desc) {
- super(desc);
- }
-
- @Override
- public void write(int repetitionLevel, byte[] bytes) {
- column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(bytes));
- }
- }
-
- private static class ArrayDataWriter<E> extends ParquetValueWriters.RepeatedWriter<ArrayData, E> {
- private final LogicalType elementType;
-
- private ArrayDataWriter(
- int definitionLevel,
- int repetitionLevel,
- ParquetValueWriter<E> writer,
- LogicalType elementType) {
- super(definitionLevel, repetitionLevel, writer);
- this.elementType = elementType;
- }
-
- @Override
- protected Iterator<E> elements(ArrayData list) {
- return new ElementIterator<>(list);
- }
-
- private class ElementIterator<E> implements Iterator<E> {
- private final int size;
- private final ArrayData list;
- private final ArrayData.ElementGetter getter;
- private int index;
-
- private ElementIterator(ArrayData list) {
- this.list = list;
- size = list.size();
- getter = ArrayData.createElementGetter(elementType);
- index = 0;
- }
-
- @Override
- public boolean hasNext() {
- return index != size;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public E next() {
- if (index >= size) {
- throw new NoSuchElementException();
- }
-
- E element = (E) getter.getElementOrNull(list, index);
- index += 1;
-
- return element;
- }
- }
- }
-
- private static class MapDataWriter<K, V>
- extends ParquetValueWriters.RepeatedKeyValueWriter<MapData, K, V> {
- private final LogicalType keyType;
- private final LogicalType valueType;
-
- private MapDataWriter(
- int definitionLevel,
- int repetitionLevel,
- ParquetValueWriter<K> keyWriter,
- ParquetValueWriter<V> valueWriter,
- LogicalType keyType,
- LogicalType valueType) {
- super(definitionLevel, repetitionLevel, keyWriter, valueWriter);
- this.keyType = keyType;
- this.valueType = valueType;
- }
-
- @Override
- protected Iterator<Map.Entry<K, V>> pairs(MapData map) {
- return new EntryIterator<>(map);
- }
-
- private class EntryIterator<K, V> implements Iterator<Map.Entry<K, V>> {
- private final int size;
- private final ArrayData keys;
- private final ArrayData values;
- private final ParquetValueReaders.ReusableEntry<K, V> entry;
- private final ArrayData.ElementGetter keyGetter;
- private final ArrayData.ElementGetter valueGetter;
- private int index;
-
- private EntryIterator(MapData map) {
- size = map.size();
- keys = map.keyArray();
- values = map.valueArray();
- entry = new ParquetValueReaders.ReusableEntry<>();
- keyGetter = ArrayData.createElementGetter(keyType);
- valueGetter = ArrayData.createElementGetter(valueType);
- index = 0;
- }
-
- @Override
- public boolean hasNext() {
- return index != size;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Map.Entry<K, V> next() {
- if (index >= size) {
- throw new NoSuchElementException();
- }
-
- entry.set(
- (K) keyGetter.getElementOrNull(keys, index),
- (V) valueGetter.getElementOrNull(values, index));
- index += 1;
-
- return entry;
- }
- }
- }
-
- private static class RowDataWriter extends ParquetValueWriters.StructWriter<RowData> {
- private final RowData.FieldGetter[] fieldGetter;
-
- RowDataWriter(List<ParquetValueWriter<?>> writers, List<LogicalType> types) {
- super(writers);
- fieldGetter = new RowData.FieldGetter[types.size()];
- for (int i = 0; i < types.size(); i += 1) {
- fieldGetter[i] = RowData.createFieldGetter(types.get(i), i);
- }
- }
-
- @Override
- protected Object get(RowData struct, int index) {
- return fieldGetter[index].getFieldOrNull(struct);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkSchemaVisitor.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkSchemaVisitor.java
deleted file mode 100644
index ba4e1a7a7a..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkSchemaVisitor.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.util.List;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.MapType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-
-abstract class FlinkSchemaVisitor<T> {
-
- static <T> T visit(RowType flinkType, Schema schema, FlinkSchemaVisitor<T> visitor) {
- return visit(flinkType, schema.asStruct(), visitor);
- }
-
- private static <T> T visit(LogicalType flinkType, Type iType, FlinkSchemaVisitor<T> visitor) {
- switch (iType.typeId()) {
- case STRUCT:
- return visitRecord(flinkType, iType.asStructType(), visitor);
-
- case MAP:
- MapType mapType = (MapType) flinkType;
- Types.MapType iMapType = iType.asMapType();
- T key;
- T value;
-
- Types.NestedField keyField = iMapType.field(iMapType.keyId());
- visitor.beforeMapKey(keyField);
- try {
- key = visit(mapType.getKeyType(), iMapType.keyType(), visitor);
- } finally {
- visitor.afterMapKey(keyField);
- }
-
- Types.NestedField valueField = iMapType.field(iMapType.valueId());
- visitor.beforeMapValue(valueField);
- try {
- value = visit(mapType.getValueType(), iMapType.valueType(), visitor);
- } finally {
- visitor.afterMapValue(valueField);
- }
-
- return visitor.map(iMapType, key, value, mapType.getKeyType(), mapType.getValueType());
-
- case LIST:
- ArrayType listType = (ArrayType) flinkType;
- Types.ListType iListType = iType.asListType();
- T element;
-
- Types.NestedField elementField = iListType.field(iListType.elementId());
- visitor.beforeListElement(elementField);
- try {
- element = visit(listType.getElementType(), iListType.elementType(), visitor);
- } finally {
- visitor.afterListElement(elementField);
- }
-
- return visitor.list(iListType, element, listType.getElementType());
-
- default:
- return visitor.primitive(iType.asPrimitiveType(), flinkType);
- }
- }
-
- private static <T> T visitRecord(
- LogicalType flinkType, Types.StructType struct, FlinkSchemaVisitor<T> visitor) {
- Preconditions.checkArgument(flinkType instanceof RowType, "%s is not a RowType.", flinkType);
- RowType rowType = (RowType) flinkType;
-
- int fieldSize = struct.fields().size();
- List<T> results = Lists.newArrayListWithExpectedSize(fieldSize);
- List<LogicalType> fieldTypes = Lists.newArrayListWithExpectedSize(fieldSize);
- List<Types.NestedField> nestedFields = struct.fields();
-
- for (int i = 0; i < fieldSize; i++) {
- Types.NestedField iField = nestedFields.get(i);
- int fieldIndex = rowType.getFieldIndex(iField.name());
- Preconditions.checkArgument(
- fieldIndex >= 0, "NestedField: %s is not found in flink RowType: %s", iField, rowType);
-
- LogicalType fieldFlinkType = rowType.getTypeAt(fieldIndex);
-
- fieldTypes.add(fieldFlinkType);
-
- visitor.beforeField(iField);
- try {
- results.add(visit(fieldFlinkType, iField.type(), visitor));
- } finally {
- visitor.afterField(iField);
- }
- }
-
- return visitor.record(struct, results, fieldTypes);
- }
-
- public T record(Types.StructType iStruct, List<T> results, List<LogicalType> fieldTypes) {
- return null;
- }
-
- public T list(Types.ListType iList, T element, LogicalType elementType) {
- return null;
- }
-
- public T map(Types.MapType iMap, T key, T value, LogicalType keyType, LogicalType valueType) {
- return null;
- }
-
- public T primitive(Type.PrimitiveType iPrimitive, LogicalType flinkPrimitive) {
- return null;
- }
-
- public void beforeField(Types.NestedField field) {}
-
- public void afterField(Types.NestedField field) {}
-
- public void beforeListElement(Types.NestedField elementField) {
- beforeField(elementField);
- }
-
- public void afterListElement(Types.NestedField elementField) {
- afterField(elementField);
- }
-
- public void beforeMapKey(Types.NestedField keyField) {
- beforeField(keyField);
- }
-
- public void afterMapKey(Types.NestedField keyField) {
- afterField(keyField);
- }
-
- public void beforeMapValue(Types.NestedField valueField) {
- beforeField(valueField);
- }
-
- public void afterMapValue(Types.NestedField valueField) {
- afterField(valueField);
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueReaders.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueReaders.java
deleted file mode 100644
index 32f6c3a2cc..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueReaders.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
-import org.apache.avro.io.Decoder;
-import org.apache.avro.util.Utf8;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.data.GenericMapData;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.iceberg.avro.ValueReader;
-import org.apache.iceberg.avro.ValueReaders;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.types.Types;
-
-public class FlinkValueReaders {
-
- private FlinkValueReaders() {}
-
- static ValueReader<StringData> strings() {
- return StringReader.INSTANCE;
- }
-
- static ValueReader<StringData> enums(List<String> symbols) {
- return new EnumReader(symbols);
- }
-
- static ValueReader<byte[]> uuids() {
- return ValueReaders.fixed(16);
- }
-
- static ValueReader<Integer> timeMicros() {
- return TimeMicrosReader.INSTANCE;
- }
-
- static ValueReader<TimestampData> timestampMills() {
- return TimestampMillsReader.INSTANCE;
- }
-
- static ValueReader<TimestampData> timestampMicros() {
- return TimestampMicrosReader.INSTANCE;
- }
-
- static ValueReader<DecimalData> decimal(
- ValueReader<byte[]> unscaledReader, int precision, int scale) {
- return new DecimalReader(unscaledReader, precision, scale);
- }
-
- static ValueReader<ArrayData> array(ValueReader<?> elementReader) {
- return new ArrayReader(elementReader);
- }
-
- static ValueReader<MapData> arrayMap(ValueReader<?> keyReader, ValueReader<?> valueReader) {
- return new ArrayMapReader(keyReader, valueReader);
- }
-
- static ValueReader<MapData> map(ValueReader<?> keyReader, ValueReader<?> valueReader) {
- return new MapReader(keyReader, valueReader);
- }
-
- static ValueReader<RowData> struct(
- List<ValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
- return new StructReader(readers, struct, idToConstant);
- }
-
- private static class StringReader implements ValueReader<StringData> {
- private static final StringReader INSTANCE = new StringReader();
-
- private StringReader() {}
-
- @Override
- public StringData read(Decoder decoder, Object reuse) throws IOException {
- // use the decoder's readString(Utf8) method because it may be a resolving decoder
- Utf8 utf8 = null;
- if (reuse instanceof StringData) {
- utf8 = new Utf8(((StringData) reuse).toBytes());
- }
-
- Utf8 string = decoder.readString(utf8);
- return StringData.fromBytes(string.getBytes(), 0, string.getByteLength());
- }
- }
-
- private static class EnumReader implements ValueReader<StringData> {
- private final StringData[] symbols;
-
- private EnumReader(List<String> symbols) {
- this.symbols = new StringData[symbols.size()];
- for (int i = 0; i < this.symbols.length; i += 1) {
- this.symbols[i] = StringData.fromBytes(symbols.get(i).getBytes(StandardCharsets.UTF_8));
- }
- }
-
- @Override
- public StringData read(Decoder decoder, Object ignore) throws IOException {
- int index = decoder.readEnum();
- return symbols[index];
- }
- }
-
- private static class DecimalReader implements ValueReader<DecimalData> {
- private final ValueReader<byte[]> bytesReader;
- private final int precision;
- private final int scale;
-
- private DecimalReader(ValueReader<byte[]> bytesReader, int precision, int scale) {
- this.bytesReader = bytesReader;
- this.precision = precision;
- this.scale = scale;
- }
-
- @Override
- public DecimalData read(Decoder decoder, Object reuse) throws IOException {
- byte[] bytes = bytesReader.read(decoder, null);
- return DecimalData.fromBigDecimal(
- new BigDecimal(new BigInteger(bytes), scale), precision, scale);
- }
- }
-
- private static class TimeMicrosReader implements ValueReader<Integer> {
- private static final TimeMicrosReader INSTANCE = new TimeMicrosReader();
-
- @Override
- public Integer read(Decoder decoder, Object reuse) throws IOException {
- long micros = decoder.readLong();
- // Flink only support time mills, just erase micros.
- return (int) (micros / 1000);
- }
- }
-
- private static class TimestampMillsReader implements ValueReader<TimestampData> {
- private static final TimestampMillsReader INSTANCE = new TimestampMillsReader();
-
- @Override
- public TimestampData read(Decoder decoder, Object reuse) throws IOException {
- return TimestampData.fromEpochMillis(decoder.readLong());
- }
- }
-
- private static class TimestampMicrosReader implements ValueReader<TimestampData> {
- private static final TimestampMicrosReader INSTANCE = new TimestampMicrosReader();
-
- @Override
- public TimestampData read(Decoder decoder, Object reuse) throws IOException {
- long micros = decoder.readLong();
- long mills = micros / 1000;
- int nanos = ((int) (micros % 1000)) * 1000;
- if (nanos < 0) {
- nanos += 1_000_000;
- mills -= 1;
- }
- return TimestampData.fromEpochMillis(mills, nanos);
- }
- }
-
- private static class ArrayReader implements ValueReader<ArrayData> {
- private final ValueReader<?> elementReader;
- private final List<Object> reusedList = Lists.newArrayList();
-
- private ArrayReader(ValueReader<?> elementReader) {
- this.elementReader = elementReader;
- }
-
- @Override
- public GenericArrayData read(Decoder decoder, Object reuse) throws IOException {
- reusedList.clear();
- long chunkLength = decoder.readArrayStart();
-
- while (chunkLength > 0) {
- for (int i = 0; i < chunkLength; i += 1) {
- reusedList.add(elementReader.read(decoder, null));
- }
-
- chunkLength = decoder.arrayNext();
- }
-
- // this will convert the list to an array so it is okay to reuse the list
- return new GenericArrayData(reusedList.toArray());
- }
- }
-
- private static MapData kvArrayToMap(List<Object> keyList, List<Object> valueList) {
- Map<Object, Object> map = Maps.newHashMap();
- Object[] keys = keyList.toArray();
- Object[] values = valueList.toArray();
- for (int i = 0; i < keys.length; i++) {
- map.put(keys[i], values[i]);
- }
-
- return new GenericMapData(map);
- }
-
- private static class ArrayMapReader implements ValueReader<MapData> {
- private final ValueReader<?> keyReader;
- private final ValueReader<?> valueReader;
-
- private final List<Object> reusedKeyList = Lists.newArrayList();
- private final List<Object> reusedValueList = Lists.newArrayList();
-
- private ArrayMapReader(ValueReader<?> keyReader, ValueReader<?> valueReader) {
- this.keyReader = keyReader;
- this.valueReader = valueReader;
- }
-
- @Override
- public MapData read(Decoder decoder, Object reuse) throws IOException {
- reusedKeyList.clear();
- reusedValueList.clear();
-
- long chunkLength = decoder.readArrayStart();
-
- while (chunkLength > 0) {
- for (int i = 0; i < chunkLength; i += 1) {
- reusedKeyList.add(keyReader.read(decoder, null));
- reusedValueList.add(valueReader.read(decoder, null));
- }
-
- chunkLength = decoder.arrayNext();
- }
-
- return kvArrayToMap(reusedKeyList, reusedValueList);
- }
- }
-
- private static class MapReader implements ValueReader<MapData> {
- private final ValueReader<?> keyReader;
- private final ValueReader<?> valueReader;
-
- private final List<Object> reusedKeyList = Lists.newArrayList();
- private final List<Object> reusedValueList = Lists.newArrayList();
-
- private MapReader(ValueReader<?> keyReader, ValueReader<?> valueReader) {
- this.keyReader = keyReader;
- this.valueReader = valueReader;
- }
-
- @Override
- public MapData read(Decoder decoder, Object reuse) throws IOException {
- reusedKeyList.clear();
- reusedValueList.clear();
-
- long chunkLength = decoder.readMapStart();
-
- while (chunkLength > 0) {
- for (int i = 0; i < chunkLength; i += 1) {
- reusedKeyList.add(keyReader.read(decoder, null));
- reusedValueList.add(valueReader.read(decoder, null));
- }
-
- chunkLength = decoder.mapNext();
- }
-
- return kvArrayToMap(reusedKeyList, reusedValueList);
- }
- }
-
- private static class StructReader extends ValueReaders.StructReader<RowData> {
- private final int numFields;
-
- private StructReader(
- List<ValueReader<?>> readers, Types.StructType struct, Map<Integer, ?> idToConstant) {
- super(readers, struct, idToConstant);
- this.numFields = readers.size();
- }
-
- @Override
- protected RowData reuseOrCreate(Object reuse) {
- if (reuse instanceof GenericRowData && ((GenericRowData) reuse).getArity() == numFields) {
- return (GenericRowData) reuse;
- }
- return new GenericRowData(numFields);
- }
-
- @Override
- protected Object get(RowData struct, int pos) {
- return null;
- }
-
- @Override
- protected void set(RowData struct, int pos, Object value) {
- ((GenericRowData) struct).setField(pos, value);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueWriters.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueWriters.java
deleted file mode 100644
index 4e86ecce28..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/FlinkValueWriters.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.io.IOException;
-import java.lang.reflect.Array;
-import java.util.List;
-import org.apache.avro.io.Encoder;
-import org.apache.avro.util.Utf8;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.iceberg.avro.ValueWriter;
-import org.apache.iceberg.types.TypeUtil;
-import org.apache.iceberg.util.DecimalUtil;
-
-public class FlinkValueWriters {
-
- private FlinkValueWriters() {}
-
- static ValueWriter<StringData> strings() {
- return StringWriter.INSTANCE;
- }
-
- static ValueWriter<Integer> timeMicros() {
- return TimeMicrosWriter.INSTANCE;
- }
-
- static ValueWriter<TimestampData> timestampMicros() {
- return TimestampMicrosWriter.INSTANCE;
- }
-
- static ValueWriter<DecimalData> decimal(int precision, int scale) {
- return new DecimalWriter(precision, scale);
- }
-
- static <T> ValueWriter<ArrayData> array(ValueWriter<T> elementWriter, LogicalType elementType) {
- return new ArrayWriter<>(elementWriter, elementType);
- }
-
- static <K, V> ValueWriter<MapData> arrayMap(
- ValueWriter<K> keyWriter,
- LogicalType keyType,
- ValueWriter<V> valueWriter,
- LogicalType valueType) {
- return new ArrayMapWriter<>(keyWriter, keyType, valueWriter, valueType);
- }
-
- static <K, V> ValueWriter<MapData> map(
- ValueWriter<K> keyWriter,
- LogicalType keyType,
- ValueWriter<V> valueWriter,
- LogicalType valueType) {
- return new MapWriter<>(keyWriter, keyType, valueWriter, valueType);
- }
-
- static ValueWriter<RowData> row(List<ValueWriter<?>> writers, List<LogicalType> types) {
- return new RowWriter(writers, types);
- }
-
- private static class StringWriter implements ValueWriter<StringData> {
- private static final StringWriter INSTANCE = new StringWriter();
-
- private StringWriter() {}
-
- @Override
- public void write(StringData s, Encoder encoder) throws IOException {
- // toBytes is cheaper than Avro calling toString, which incurs encoding costs
- encoder.writeString(new Utf8(s.toBytes()));
- }
- }
-
- private static class DecimalWriter implements ValueWriter<DecimalData> {
- private final int precision;
- private final int scale;
- private final ThreadLocal<byte[]> bytes;
-
- private DecimalWriter(int precision, int scale) {
- this.precision = precision;
- this.scale = scale;
- this.bytes =
- ThreadLocal.withInitial(() -> new byte[TypeUtil.decimalRequiredBytes(precision)]);
- }
-
- @Override
- public void write(DecimalData d, Encoder encoder) throws IOException {
- encoder.writeFixed(
- DecimalUtil.toReusedFixLengthBytes(precision, scale, d.toBigDecimal(), bytes.get()));
- }
- }
-
- private static class TimeMicrosWriter implements ValueWriter<Integer> {
- private static final TimeMicrosWriter INSTANCE = new TimeMicrosWriter();
-
- @Override
- public void write(Integer timeMills, Encoder encoder) throws IOException {
- encoder.writeLong(timeMills * 1000L);
- }
- }
-
- private static class TimestampMicrosWriter implements ValueWriter<TimestampData> {
- private static final TimestampMicrosWriter INSTANCE = new TimestampMicrosWriter();
-
- @Override
- public void write(TimestampData timestampData, Encoder encoder) throws IOException {
- long micros =
- timestampData.getMillisecond() * 1000 + timestampData.getNanoOfMillisecond() / 1000;
- encoder.writeLong(micros);
- }
- }
-
- private static class ArrayWriter<T> implements ValueWriter<ArrayData> {
- private final ValueWriter<T> elementWriter;
- private final ArrayData.ElementGetter elementGetter;
-
- private ArrayWriter(ValueWriter<T> elementWriter, LogicalType elementType) {
- this.elementWriter = elementWriter;
- this.elementGetter = ArrayData.createElementGetter(elementType);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void write(ArrayData array, Encoder encoder) throws IOException {
- encoder.writeArrayStart();
- int numElements = array.size();
- encoder.setItemCount(numElements);
- for (int i = 0; i < numElements; i += 1) {
- encoder.startItem();
- elementWriter.write((T) elementGetter.getElementOrNull(array, i), encoder);
- }
- encoder.writeArrayEnd();
- }
- }
-
- private static class ArrayMapWriter<K, V> implements ValueWriter<MapData> {
- private final ValueWriter<K> keyWriter;
- private final ValueWriter<V> valueWriter;
- private final ArrayData.ElementGetter keyGetter;
- private final ArrayData.ElementGetter valueGetter;
-
- private ArrayMapWriter(
- ValueWriter<K> keyWriter,
- LogicalType keyType,
- ValueWriter<V> valueWriter,
- LogicalType valueType) {
- this.keyWriter = keyWriter;
- this.keyGetter = ArrayData.createElementGetter(keyType);
- this.valueWriter = valueWriter;
- this.valueGetter = ArrayData.createElementGetter(valueType);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void write(MapData map, Encoder encoder) throws IOException {
- encoder.writeArrayStart();
- int numElements = map.size();
- encoder.setItemCount(numElements);
- ArrayData keyArray = map.keyArray();
- ArrayData valueArray = map.valueArray();
- for (int i = 0; i < numElements; i += 1) {
- encoder.startItem();
- keyWriter.write((K) keyGetter.getElementOrNull(keyArray, i), encoder);
- valueWriter.write((V) valueGetter.getElementOrNull(valueArray, i), encoder);
- }
- encoder.writeArrayEnd();
- }
- }
-
- private static class MapWriter<K, V> implements ValueWriter<MapData> {
- private final ValueWriter<K> keyWriter;
- private final ValueWriter<V> valueWriter;
- private final ArrayData.ElementGetter keyGetter;
- private final ArrayData.ElementGetter valueGetter;
-
- private MapWriter(
- ValueWriter<K> keyWriter,
- LogicalType keyType,
- ValueWriter<V> valueWriter,
- LogicalType valueType) {
- this.keyWriter = keyWriter;
- this.keyGetter = ArrayData.createElementGetter(keyType);
- this.valueWriter = valueWriter;
- this.valueGetter = ArrayData.createElementGetter(valueType);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void write(MapData map, Encoder encoder) throws IOException {
- encoder.writeMapStart();
- int numElements = map.size();
- encoder.setItemCount(numElements);
- ArrayData keyArray = map.keyArray();
- ArrayData valueArray = map.valueArray();
- for (int i = 0; i < numElements; i += 1) {
- encoder.startItem();
- keyWriter.write((K) keyGetter.getElementOrNull(keyArray, i), encoder);
- valueWriter.write((V) valueGetter.getElementOrNull(valueArray, i), encoder);
- }
- encoder.writeMapEnd();
- }
- }
-
- static class RowWriter implements ValueWriter<RowData> {
- private final ValueWriter<?>[] writers;
- private final RowData.FieldGetter[] getters;
-
- private RowWriter(List<ValueWriter<?>> writers, List<LogicalType> types) {
- this.writers = (ValueWriter<?>[]) Array.newInstance(ValueWriter.class, writers.size());
- this.getters = new RowData.FieldGetter[writers.size()];
- for (int i = 0; i < writers.size(); i += 1) {
- this.writers[i] = writers.get(i);
- this.getters[i] = RowData.createFieldGetter(types.get(i), i);
- }
- }
-
- @Override
- public void write(RowData row, Encoder encoder) throws IOException {
- for (int i = 0; i < writers.length; i += 1) {
- if (row.isNullAt(i)) {
- writers[i].write(null, encoder);
- } else {
- write(row, i, writers[i], encoder);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- private <T> void write(RowData row, int pos, ValueWriter<T> writer, Encoder encoder)
- throws IOException {
- writer.write((T) getters[pos].getFieldOrNull(row), encoder);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java
deleted file mode 100644
index 33feb2e321..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/ParquetWithFlinkSchemaVisitor.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.util.Deque;
-import java.util.List;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.MapType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.logical.RowType.RowField;
-import org.apache.iceberg.avro.AvroSchemaUtil;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.OriginalType;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-
-public class ParquetWithFlinkSchemaVisitor<T> {
- private final Deque<String> fieldNames = Lists.newLinkedList();
-
- public static <T> T visit(
- LogicalType sType, Type type, ParquetWithFlinkSchemaVisitor<T> visitor) {
- Preconditions.checkArgument(sType != null, "Invalid DataType: null");
- if (type instanceof MessageType) {
- Preconditions.checkArgument(
- sType instanceof RowType, "Invalid struct: %s is not a struct", sType);
- RowType struct = (RowType) sType;
- return visitor.message(
- struct, (MessageType) type, visitFields(struct, type.asGroupType(), visitor));
- } else if (type.isPrimitive()) {
- return visitor.primitive(sType, type.asPrimitiveType());
- } else {
- // if not a primitive, the typeId must be a group
- GroupType group = type.asGroupType();
- OriginalType annotation = group.getOriginalType();
- if (annotation != null) {
- switch (annotation) {
- case LIST:
- Preconditions.checkArgument(
- !group.isRepetition(Type.Repetition.REPEATED),
- "Invalid list: top-level group is repeated: %s",
- group);
- Preconditions.checkArgument(
- group.getFieldCount() == 1,
- "Invalid list: does not contain single repeated field: %s",
- group);
-
- GroupType repeatedElement = group.getFields().get(0).asGroupType();
- Preconditions.checkArgument(
- repeatedElement.isRepetition(Type.Repetition.REPEATED),
- "Invalid list: inner group is not repeated");
- Preconditions.checkArgument(
- repeatedElement.getFieldCount() <= 1,
- "Invalid list: repeated group is not a single field: %s",
- group);
-
- Preconditions.checkArgument(
- sType instanceof ArrayType, "Invalid list: %s is not an array", sType);
- ArrayType array = (ArrayType) sType;
- RowType.RowField element =
- new RowField(
- "element", array.getElementType(), "element of " + array.asSummaryString());
-
- visitor.fieldNames.push(repeatedElement.getName());
- try {
- T elementResult = null;
- if (repeatedElement.getFieldCount() > 0) {
- elementResult = visitField(element, repeatedElement.getType(0), visitor);
- }
-
- return visitor.list(array, group, elementResult);
-
- } finally {
- visitor.fieldNames.pop();
- }
-
- case MAP:
- Preconditions.checkArgument(
- !group.isRepetition(Type.Repetition.REPEATED),
- "Invalid map: top-level group is repeated: %s",
- group);
- Preconditions.checkArgument(
- group.getFieldCount() == 1,
- "Invalid map: does not contain single repeated field: %s",
- group);
-
- GroupType repeatedKeyValue = group.getType(0).asGroupType();
- Preconditions.checkArgument(
- repeatedKeyValue.isRepetition(Type.Repetition.REPEATED),
- "Invalid map: inner group is not repeated");
- Preconditions.checkArgument(
- repeatedKeyValue.getFieldCount() <= 2,
- "Invalid map: repeated group does not have 2 fields");
-
- Preconditions.checkArgument(
- sType instanceof MapType, "Invalid map: %s is not a map", sType);
- MapType map = (MapType) sType;
- RowField keyField =
- new RowField("key", map.getKeyType(), "key of " + map.asSummaryString());
- RowField valueField =
- new RowField("value", map.getValueType(), "value of " + map.asSummaryString());
-
- visitor.fieldNames.push(repeatedKeyValue.getName());
- try {
- T keyResult = null;
- T valueResult = null;
- switch (repeatedKeyValue.getFieldCount()) {
- case 2:
- // if there are 2 fields, both key and value are projected
- keyResult = visitField(keyField, repeatedKeyValue.getType(0), visitor);
- valueResult = visitField(valueField, repeatedKeyValue.getType(1), visitor);
- break;
- case 1:
- // if there is just one, use the name to determine what it is
- Type keyOrValue = repeatedKeyValue.getType(0);
- if (keyOrValue.getName().equalsIgnoreCase("key")) {
- keyResult = visitField(keyField, keyOrValue, visitor);
- // value result remains null
- } else {
- valueResult = visitField(valueField, keyOrValue, visitor);
- // key result remains null
- }
- break;
- default:
- // both results will remain null
- }
-
- return visitor.map(map, group, keyResult, valueResult);
-
- } finally {
- visitor.fieldNames.pop();
- }
-
- default:
- }
- }
- Preconditions.checkArgument(
- sType instanceof RowType, "Invalid struct: %s is not a struct", sType);
- RowType struct = (RowType) sType;
- return visitor.struct(struct, group, visitFields(struct, group, visitor));
- }
- }
-
- private static <T> T visitField(
- RowType.RowField sField, Type field, ParquetWithFlinkSchemaVisitor<T> visitor) {
- visitor.fieldNames.push(field.getName());
- try {
- return visit(sField.getType(), field, visitor);
- } finally {
- visitor.fieldNames.pop();
- }
- }
-
- private static <T> List<T> visitFields(
- RowType struct, GroupType group, ParquetWithFlinkSchemaVisitor<T> visitor) {
- List<RowType.RowField> sFields = struct.getFields();
- Preconditions.checkArgument(
- sFields.size() == group.getFieldCount(), "Structs do not match: %s and %s", struct, group);
- List<T> results = Lists.newArrayListWithExpectedSize(group.getFieldCount());
- for (int i = 0; i < sFields.size(); i += 1) {
- Type field = group.getFields().get(i);
- RowType.RowField sField = sFields.get(i);
- Preconditions.checkArgument(
- field.getName().equals(AvroSchemaUtil.makeCompatibleName(sField.getName())),
- "Structs do not match: field %s != %s",
- field.getName(),
- sField.getName());
- results.add(visitField(sField, field, visitor));
- }
-
- return results;
- }
-
- public T message(RowType sStruct, MessageType message, List<T> fields) {
- return null;
- }
-
- public T struct(RowType sStruct, GroupType struct, List<T> fields) {
- return null;
- }
-
- public T list(ArrayType sArray, GroupType array, T element) {
- return null;
- }
-
- public T map(MapType sMap, GroupType map, T key, T value) {
- return null;
- }
-
- public T primitive(LogicalType sPrimitive, PrimitiveType primitive) {
- return null;
- }
-
- protected String[] currentPath() {
- return Lists.newArrayList(fieldNames.descendingIterator()).toArray(new String[0]);
- }
-
- protected String[] path(String name) {
- List<String> list = Lists.newArrayList(fieldNames.descendingIterator());
- list.add(name);
- return list.toArray(new String[0]);
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/RowDataProjection.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/RowDataProjection.java
deleted file mode 100644
index 33816c97ac..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/RowDataProjection.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Objects;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RawValueData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.types.RowKind;
-import org.apache.flink.util.StringUtils;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.flink.FlinkSchemaUtil;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.types.Types;
-
-public class RowDataProjection implements RowData {
- /**
- * Creates a projecting wrapper for {@link RowData} rows.
- *
- * <p>This projection will not project the nested children types of repeated types like lists and
- * maps.
- *
- * @param schema schema of rows wrapped by this projection
- * @param projectedSchema result schema of the projected rows
- * @return a wrapper to project rows
- */
- public static RowDataProjection create(Schema schema, Schema projectedSchema) {
- return RowDataProjection.create(
- FlinkSchemaUtil.convert(schema), schema.asStruct(), projectedSchema.asStruct());
- }
-
- /**
- * Creates a projecting wrapper for {@link RowData} rows.
- *
- * <p>This projection will not project the nested children types of repeated types like lists and
- * maps.
- *
- * @param rowType flink row type of rows wrapped by this projection
- * @param schema schema of rows wrapped by this projection
- * @param projectedSchema result schema of the projected rows
- * @return a wrapper to project rows
- */
- public static RowDataProjection create(
- RowType rowType, Types.StructType schema, Types.StructType projectedSchema) {
- return new RowDataProjection(rowType, schema, projectedSchema);
- }
-
- private final RowData.FieldGetter[] getters;
- private RowData rowData;
-
- private RowDataProjection(
- RowType rowType, Types.StructType rowStruct, Types.StructType projectType) {
- Map<Integer, Integer> fieldIdToPosition = Maps.newHashMap();
- for (int i = 0; i < rowStruct.fields().size(); i++) {
- fieldIdToPosition.put(rowStruct.fields().get(i).fieldId(), i);
- }
-
- this.getters = new RowData.FieldGetter[projectType.fields().size()];
- for (int i = 0; i < getters.length; i++) {
- Types.NestedField projectField = projectType.fields().get(i);
- Types.NestedField rowField = rowStruct.field(projectField.fieldId());
-
- Preconditions.checkNotNull(
- rowField,
- "Cannot locate the project field <%s> in the iceberg struct <%s>",
- projectField,
- rowStruct);
-
- getters[i] =
- createFieldGetter(
- rowType, fieldIdToPosition.get(projectField.fieldId()), rowField, projectField);
- }
- }
-
- private static RowData.FieldGetter createFieldGetter(
- RowType rowType, int position, Types.NestedField rowField, Types.NestedField projectField) {
- Preconditions.checkArgument(
- rowField.type().typeId() == projectField.type().typeId(),
- "Different iceberg type between row field <%s> and project field <%s>",
- rowField,
- projectField);
-
- switch (projectField.type().typeId()) {
- case STRUCT:
- RowType nestedRowType = (RowType) rowType.getTypeAt(position);
- return row -> {
- // null nested struct value
- if (row.isNullAt(position)) {
- return null;
- }
-
- RowData nestedRow = row.getRow(position, nestedRowType.getFieldCount());
- return RowDataProjection.create(
- nestedRowType, rowField.type().asStructType(), projectField.type().asStructType())
- .wrap(nestedRow);
- };
-
- case MAP:
- Types.MapType projectedMap = projectField.type().asMapType();
- Types.MapType originalMap = rowField.type().asMapType();
-
- boolean keyProjectable =
- !projectedMap.keyType().isNestedType()
- || projectedMap.keyType().equals(originalMap.keyType());
- boolean valueProjectable =
- !projectedMap.valueType().isNestedType()
- || projectedMap.valueType().equals(originalMap.valueType());
- Preconditions.checkArgument(
- keyProjectable && valueProjectable,
- "Cannot project a partial map key or value with non-primitive type. Trying to project <%s> out of <%s>",
- projectField,
- rowField);
-
- return RowData.createFieldGetter(rowType.getTypeAt(position), position);
-
- case LIST:
- Types.ListType projectedList = projectField.type().asListType();
- Types.ListType originalList = rowField.type().asListType();
-
- boolean elementProjectable =
- !projectedList.elementType().isNestedType()
- || projectedList.elementType().equals(originalList.elementType());
- Preconditions.checkArgument(
- elementProjectable,
- "Cannot project a partial list element with non-primitive type. Trying to project <%s> out of <%s>",
- projectField,
- rowField);
-
- return RowData.createFieldGetter(rowType.getTypeAt(position), position);
-
- default:
- return RowData.createFieldGetter(rowType.getTypeAt(position), position);
- }
- }
-
- public RowData wrap(RowData row) {
- // StructProjection allow wrapping null root struct object.
- // See more discussions in https://github.com/apache/iceberg/pull/7517.
- // RowDataProjection never allowed null root object to be wrapped.
- // Hence, it is fine to enforce strict Preconditions check here.
- Preconditions.checkArgument(row != null, "Invalid row data: null");
- this.rowData = row;
- return this;
- }
-
- private Object getValue(int pos) {
- Preconditions.checkState(rowData != null, "Row data not wrapped");
- return getters[pos].getFieldOrNull(rowData);
- }
-
- @Override
- public int getArity() {
- return getters.length;
- }
-
- @Override
- public RowKind getRowKind() {
- Preconditions.checkState(rowData != null, "Row data not wrapped");
- return rowData.getRowKind();
- }
-
- @Override
- public void setRowKind(RowKind kind) {
- throw new UnsupportedOperationException("Cannot set row kind in the RowDataProjection");
- }
-
- @Override
- public boolean isNullAt(int pos) {
- return getValue(pos) == null;
- }
-
- @Override
- public boolean getBoolean(int pos) {
- return (boolean) getValue(pos);
- }
-
- @Override
- public byte getByte(int pos) {
- return (byte) getValue(pos);
- }
-
- @Override
- public short getShort(int pos) {
- return (short) getValue(pos);
- }
-
- @Override
- public int getInt(int pos) {
- return (int) getValue(pos);
- }
-
- @Override
- public long getLong(int pos) {
- return (long) getValue(pos);
- }
-
- @Override
- public float getFloat(int pos) {
- return (float) getValue(pos);
- }
-
- @Override
- public double getDouble(int pos) {
- return (double) getValue(pos);
- }
-
- @Override
- public StringData getString(int pos) {
- return (StringData) getValue(pos);
- }
-
- @Override
- public DecimalData getDecimal(int pos, int precision, int scale) {
- return (DecimalData) getValue(pos);
- }
-
- @Override
- public TimestampData getTimestamp(int pos, int precision) {
- return (TimestampData) getValue(pos);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public <T> RawValueData<T> getRawValue(int pos) {
- return (RawValueData<T>) getValue(pos);
- }
-
- @Override
- public byte[] getBinary(int pos) {
- return (byte[]) getValue(pos);
- }
-
- @Override
- public ArrayData getArray(int pos) {
- return (ArrayData) getValue(pos);
- }
-
- @Override
- public MapData getMap(int pos) {
- return (MapData) getValue(pos);
- }
-
- @Override
- public RowData getRow(int pos, int numFields) {
- return (RowData) getValue(pos);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (!(o instanceof RowDataProjection)) {
- return false;
- }
-
- RowDataProjection that = (RowDataProjection) o;
- return deepEquals(that);
- }
-
- @Override
- public int hashCode() {
- int result = Objects.hashCode(getRowKind());
- for (int pos = 0; pos < getArity(); pos++) {
- if (!isNullAt(pos)) {
- // Arrays.deepHashCode handles array object properly
- result = 31 * result + Arrays.deepHashCode(new Object[] {getValue(pos)});
- }
- }
-
- return result;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(getRowKind().shortString()).append("(");
- for (int pos = 0; pos < getArity(); pos++) {
- if (pos != 0) {
- sb.append(",");
- }
- // copied the behavior from Flink GenericRowData
- sb.append(StringUtils.arrayAwareToString(getValue(pos)));
- }
-
- sb.append(")");
- return sb.toString();
- }
-
- private boolean deepEquals(RowDataProjection other) {
- if (getRowKind() != other.getRowKind()) {
- return false;
- }
-
- if (getArity() != other.getArity()) {
- return false;
- }
-
- for (int pos = 0; pos < getArity(); ++pos) {
- if (isNullAt(pos) && other.isNullAt(pos)) {
- continue;
- }
-
- if ((isNullAt(pos) && !other.isNullAt(pos)) || (!isNullAt(pos) && other.isNullAt(pos))) {
- return false;
- }
-
- // Objects.deepEquals handles array object properly
- if (!Objects.deepEquals(getValue(pos), other.getValue(pos))) {
- return false;
- }
- }
-
- return true;
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/RowDataUtil.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/RowDataUtil.java
deleted file mode 100644
index c5cb51b7ea..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/RowDataUtil.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.util.Utf8;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-import org.apache.iceberg.util.ByteBuffers;
-import org.apache.iceberg.util.DateTimeUtil;
-
-public class RowDataUtil {
-
- private RowDataUtil() {}
-
- public static Object convertConstant(Type type, Object value) {
- if (value == null) {
- return null;
- }
-
- switch (type.typeId()) {
- case DECIMAL: // DecimalData
- Types.DecimalType decimal = (Types.DecimalType) type;
- return DecimalData.fromBigDecimal((BigDecimal) value, decimal.precision(), decimal.scale());
- case STRING: // StringData
- if (value instanceof Utf8) {
- Utf8 utf8 = (Utf8) value;
- return StringData.fromBytes(utf8.getBytes(), 0, utf8.getByteLength());
- }
- return StringData.fromString(value.toString());
- case FIXED: // byte[]
- if (value instanceof byte[]) {
- return value;
- } else if (value instanceof GenericData.Fixed) {
- return ((GenericData.Fixed) value).bytes();
- }
- return ByteBuffers.toByteArray((ByteBuffer) value);
- case BINARY: // byte[]
- return ByteBuffers.toByteArray((ByteBuffer) value);
- case TIME: // int mills instead of long
- return (int) ((Long) value / 1000);
- case TIMESTAMP: // TimestampData
- return TimestampData.fromLocalDateTime(DateTimeUtil.timestampFromMicros((Long) value));
- default:
- }
- return value;
- }
-
- /**
- * Similar to the private {@link RowDataSerializer#copyRowData(RowData, RowData)} method. This
- * skips the check the arity of rowType and from, because the from RowData may contains additional
- * column for position deletes. Using {@link RowDataSerializer#copy(RowData, RowData)} will fail
- * the arity check.
- */
- public static RowData clone(
- RowData from, RowData reuse, RowType rowType, TypeSerializer[] fieldSerializers) {
- GenericRowData ret;
- if (reuse instanceof GenericRowData) {
- ret = (GenericRowData) reuse;
- } else {
- ret = new GenericRowData(from.getArity());
- }
- ret.setRowKind(from.getRowKind());
- for (int i = 0; i < rowType.getFieldCount(); i++) {
- if (!from.isNullAt(i)) {
- RowData.FieldGetter getter = RowData.createFieldGetter(rowType.getTypeAt(i), i);
- ret.setField(i, fieldSerializers[i].copy(getter.getFieldOrNull(from)));
- } else {
- ret.setField(i, null);
- }
- }
- return ret;
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/StructRowData.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/StructRowData.java
deleted file mode 100644
index 1019285018..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/data/StructRowData.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.data;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.time.Duration;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.data.GenericMapData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RawValueData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.types.RowKind;
-import org.apache.iceberg.StructLike;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-import org.apache.iceberg.util.ByteBuffers;
-
-@Internal
-public class StructRowData implements RowData {
- private final Types.StructType type;
- private RowKind kind;
- private StructLike struct;
-
- public StructRowData(Types.StructType type) {
- this(type, RowKind.INSERT);
- }
-
- public StructRowData(Types.StructType type, RowKind kind) {
- this(type, null, kind);
- }
-
- private StructRowData(Types.StructType type, StructLike struct) {
- this(type, struct, RowKind.INSERT);
- }
-
- private StructRowData(Types.StructType type, StructLike struct, RowKind kind) {
- this.type = type;
- this.struct = struct;
- this.kind = kind;
- }
-
- public StructRowData setStruct(StructLike newStruct) {
- this.struct = newStruct;
- return this;
- }
-
- @Override
- public int getArity() {
- return struct.size();
- }
-
- @Override
- public RowKind getRowKind() {
- return kind;
- }
-
- @Override
- public void setRowKind(RowKind newKind) {
- Preconditions.checkNotNull(newKind, "kind can not be null");
- this.kind = newKind;
- }
-
- @Override
- public boolean isNullAt(int pos) {
- return struct.get(pos, Object.class) == null;
- }
-
- @Override
- public boolean getBoolean(int pos) {
- return struct.get(pos, Boolean.class);
- }
-
- @Override
- public byte getByte(int pos) {
- return (byte) (int) struct.get(pos, Integer.class);
- }
-
- @Override
- public short getShort(int pos) {
- return (short) (int) struct.get(pos, Integer.class);
- }
-
- @Override
- public int getInt(int pos) {
- Object integer = struct.get(pos, Object.class);
-
- if (integer instanceof Integer) {
- return (int) integer;
- } else if (integer instanceof LocalDate) {
- return (int) ((LocalDate) integer).toEpochDay();
- } else if (integer instanceof LocalTime) {
- return (int) (((LocalTime) integer).toNanoOfDay() / 1000_000);
- } else {
- throw new IllegalStateException(
- "Unknown type for int field. Type name: " + integer.getClass().getName());
- }
- }
-
- @Override
- public long getLong(int pos) {
- Object longVal = struct.get(pos, Object.class);
-
- if (longVal instanceof Long) {
- return (long) longVal;
- } else if (longVal instanceof OffsetDateTime) {
- return Duration.between(Instant.EPOCH, (OffsetDateTime) longVal).toNanos() / 1000;
- } else if (longVal instanceof LocalDate) {
- return ((LocalDate) longVal).toEpochDay();
- } else if (longVal instanceof LocalTime) {
- return ((LocalTime) longVal).toNanoOfDay();
- } else if (longVal instanceof LocalDateTime) {
- return Duration.between(Instant.EPOCH, ((LocalDateTime) longVal).atOffset(ZoneOffset.UTC))
- .toNanos()
- / 1000;
- } else {
- throw new IllegalStateException(
- "Unknown type for long field. Type name: " + longVal.getClass().getName());
- }
- }
-
- @Override
- public float getFloat(int pos) {
- return struct.get(pos, Float.class);
- }
-
- @Override
- public double getDouble(int pos) {
- return struct.get(pos, Double.class);
- }
-
- @Override
- public StringData getString(int pos) {
- return isNullAt(pos) ? null : getStringDataInternal(pos);
- }
-
- private StringData getStringDataInternal(int pos) {
- CharSequence seq = struct.get(pos, CharSequence.class);
- return StringData.fromString(seq.toString());
- }
-
- @Override
- public DecimalData getDecimal(int pos, int precision, int scale) {
- return isNullAt(pos)
- ? null
- : DecimalData.fromBigDecimal(getDecimalInternal(pos), precision, scale);
- }
-
- private BigDecimal getDecimalInternal(int pos) {
- return struct.get(pos, BigDecimal.class);
- }
-
- @Override
- public TimestampData getTimestamp(int pos, int precision) {
- long timeLong = getLong(pos);
- return TimestampData.fromEpochMillis(timeLong / 1000, (int) (timeLong % 1000) * 1000);
- }
-
- @Override
- public <T> RawValueData<T> getRawValue(int pos) {
- throw new UnsupportedOperationException("Not supported yet.");
- }
-
- @Override
- public byte[] getBinary(int pos) {
- return isNullAt(pos) ? null : getBinaryInternal(pos);
- }
-
- private byte[] getBinaryInternal(int pos) {
- Object bytes = struct.get(pos, Object.class);
-
- // should only be either ByteBuffer or byte[]
- if (bytes instanceof ByteBuffer) {
- return ByteBuffers.toByteArray((ByteBuffer) bytes);
- } else if (bytes instanceof byte[]) {
- return (byte[]) bytes;
- } else if (bytes instanceof UUID) {
- UUID uuid = (UUID) bytes;
- ByteBuffer bb = ByteBuffer.allocate(16);
- bb.putLong(uuid.getMostSignificantBits());
- bb.putLong(uuid.getLeastSignificantBits());
- return bb.array();
- } else {
- throw new IllegalStateException(
- "Unknown type for binary field. Type name: " + bytes.getClass().getName());
- }
- }
-
- @Override
- public ArrayData getArray(int pos) {
- return isNullAt(pos)
- ? null
- : (ArrayData)
- convertValue(type.fields().get(pos).type().asListType(), struct.get(pos, List.class));
- }
-
- @Override
- public MapData getMap(int pos) {
- return isNullAt(pos)
- ? null
- : (MapData)
- convertValue(type.fields().get(pos).type().asMapType(), struct.get(pos, Map.class));
- }
-
- @Override
- public RowData getRow(int pos, int numFields) {
- return isNullAt(pos) ? null : getStructRowData(pos, numFields);
- }
-
- private StructRowData getStructRowData(int pos, int numFields) {
- return new StructRowData(
- type.fields().get(pos).type().asStructType(), struct.get(pos, StructLike.class));
- }
-
- private Object convertValue(Type elementType, Object value) {
- switch (elementType.typeId()) {
- case BOOLEAN:
- case INTEGER:
- case DATE:
- case TIME:
- case LONG:
- case FLOAT:
- case DOUBLE:
- case DECIMAL:
- return value;
- case TIMESTAMP:
- long millisecond = (long) value / 1000;
- int nanoOfMillisecond = (int) ((Long) value % 1000) * 1000;
- return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond);
- case STRING:
- return StringData.fromString(value.toString());
- case FIXED:
- case BINARY:
- return ByteBuffers.toByteArray((ByteBuffer) value);
- case STRUCT:
- return new StructRowData(elementType.asStructType(), (StructLike) value);
- case LIST:
- List<Object> list = (List<Object>) value;
- Object[] array = new Object[list.size()];
-
- int index = 0;
- for (Object element : list) {
- if (element == null) {
- array[index] = null;
- } else {
- array[index] = convertValue(elementType.asListType().elementType(), element);
- }
-
- index += 1;
- }
- return new GenericArrayData(array);
- case MAP:
- Types.MapType mapType = elementType.asMapType();
- Set<? extends Map.Entry<?, ?>> entries = ((Map<?, ?>) value).entrySet();
- Map<Object, Object> result = Maps.newHashMap();
- for (Map.Entry<?, ?> entry : entries) {
- final Object keyValue = convertValue(mapType.keyType(), entry.getKey());
- final Object valueValue = convertValue(mapType.valueType(), entry.getValue());
- result.put(keyValue, valueValue);
- }
-
- return new GenericMapData(result);
- default:
- throw new UnsupportedOperationException("Unsupported element type: " + elementType);
- }
- }
-}
diff --git a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/AvroGenericRecordToRowDataMapper.java b/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/AvroGenericRecordToRowDataMapper.java
deleted file mode 100644
index 04e168385a..0000000000
--- a/flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/AvroGenericRecordToRowDataMapper.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.flink.sink;
-
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.formats.avro.AvroToRowDataConverters;
-import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.utils.TypeConversions;
-import org.apache.iceberg.avro.AvroSchemaUtil;
-
-/**
- * This util class converts Avro GenericRecord to Flink RowData. <br>
- * <br>
- * Internally it uses Flink {@link AvroToRowDataConverters}. Because of the precision difference
- * between how Iceberg schema (micro) and Flink {@link AvroToRowDataConverters} (milli) deal with
- * time type, we can't directly use the Avro Schema converted from Iceberg schema via {@link
- * AvroSchemaUtil#convert(org.apache.iceberg.Schema, String)}.
- */
-public class AvroGenericRecordToRowDataMapper implements MapFunction<GenericRecord, RowData> {
-
- private final AvroToRowDataConverters.AvroToRowDataConverter converter;
-
- AvroGenericRecordToRowDataMapper(RowType rowType) {
- this.converter = AvroToRowDataConverters.createRowConverter(rowType);
- }
-
- @Override
- public RowData map(GenericRecord genericRecord) throws Exception {
- return (RowData) converter.convert(genericRecord);
- }
-
- /** Create a mapper based on Avro schema. */
- public static AvroGenericRecordToRowDataMapper forAvroSchema(Schema avroSchema) {
- DataType dataType = AvroSchemaConverter.convertToDataType(avroSchema.toString());
- LogicalType logicalType = TypeConversions.fromDataToLogicalType(dataType);
- RowType rowType = RowType.of(logicalType.getChildren().stream().toArray(LogicalType[]::new));
... 42366 lines suppressed ...