You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/06/28 16:15:02 UTC
[iceberg] 03/04: copy 3.2 files from 3.3
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit e28b28831de81564cbbdda36aee5d3fdace893ea
Author: Prashant Singh <ps...@amazon.com>
AuthorDate: Tue Jun 28 10:31:13 2022 +0530
copy 3.2 files from 3.3
---
spark/v3.2/build.gradle | 279 +++
.../IcebergSqlExtensions.g4 | 293 +++
.../extensions/IcebergSparkSessionExtensions.scala | 77 +
.../spark/sql/catalyst/ProjectingInternalRow.scala | 123 ++
.../analysis/AlignRowLevelCommandAssignments.scala | 113 ++
.../AlignedRowLevelIcebergCommandCheck.scala | 38 +
.../analysis/AssignmentAlignmentSupport.scala | 202 ++
.../analysis/CheckMergeIntoTableConditions.scala | 74 +
.../MergeIntoIcebergTableResolutionCheck.scala | 35 +
.../analysis/ProcedureArgumentCoercion.scala | 56 +
.../analysis/ResolveMergeIntoTableReferences.scala | 153 ++
.../sql/catalyst/analysis/ResolveProcedures.scala | 190 ++
.../catalyst/analysis/RewriteDeleteFromTable.scala | 118 ++
.../catalyst/analysis/RewriteMergeIntoTable.scala | 387 ++++
.../catalyst/analysis/RewriteRowLevelCommand.scala | 149 ++
.../sql/catalyst/analysis/RewriteUpdateTable.scala | 183 ++
.../sql/catalyst/expressions/AssignmentUtils.scala | 86 +
.../expressions/ExtendedV2ExpressionUtils.scala | 96 +
.../ExtendedReplaceNullWithFalseInPredicate.scala | 132 ++
.../ExtendedSimplifyConditionalsInPredicate.scala | 94 +
.../IcebergSparkSqlExtensionsParser.scala | 371 ++++
.../IcebergSqlExtensionsAstBuilder.scala | 289 +++
.../planning/RewrittenRowLevelCommand.scala | 113 ++
.../catalyst/plans/logical/AddPartitionField.scala | 33 +
.../spark/sql/catalyst/plans/logical/Call.scala | 33 +
.../plans/logical/DeleteFromIcebergTable.scala | 49 +
.../plans/logical/DropIdentifierFields.scala | 34 +
.../plans/logical/DropPartitionField.scala | 33 +
.../plans/logical/MergeIntoIcebergTable.scala | 77 +
.../sql/catalyst/plans/logical/MergeRows.scala | 56 +
.../catalyst/plans/logical/NoStatsUnaryNode.scala | 31 +
.../sql/catalyst/plans/logical/ReplaceData.scala | 69 +
.../plans/logical/ReplacePartitionField.scala | 38 +
.../catalyst/plans/logical/RowLevelCommand.scala | 28 +
.../plans/logical/SetIdentifierFields.scala | 35 +
.../logical/UnresolvedMergeIntoIcebergTable.scala | 45 +
.../plans/logical/UpdateIcebergTable.scala | 53 +
.../plans/logical/V2WriteCommandLike.scala | 42 +
.../sql/catalyst/plans/logical/WriteDelta.scala | 126 ++
.../sql/catalyst/plans/logical/statements.scala | 44 +
.../spark/sql/catalyst/util/RowDeltaUtils.scala | 27 +
.../sql/catalyst/util/WriteDeltaProjections.scala | 27 +
.../connector/expressions/TruncateTransform.scala | 38 +
.../write/ExtendedLogicalWriteInfoImpl.scala | 31 +
.../write/RowLevelOperationInfoImpl.scala | 28 +
.../connector/write/RowLevelOperationTable.scala | 54 +
.../datasources/v2/AddPartitionFieldExec.scala | 56 +
.../sql/execution/datasources/v2/CallExec.scala | 40 +
.../datasources/v2/DropIdentifierFieldsExec.scala | 65 +
.../datasources/v2/DropPartitionFieldExec.scala | 67 +
.../v2/ExtendedDataSourceV2Implicits.scala | 40 +
.../v2/ExtendedDataSourceV2Strategy.scala | 141 ++
.../v2/ExtendedDistributionAndOrderingUtils.scala | 85 +
.../datasources/v2/ExtendedV2Writes.scala | 140 ++
.../execution/datasources/v2/MergeRowsExec.scala | 190 ++
.../v2/OptimizeMetadataOnlyDeleteFromTable.scala | 73 +
.../execution/datasources/v2/ReplaceDataExec.scala | 40 +
.../datasources/v2/ReplacePartitionFieldExec.scala | 72 +
.../v2/ReplaceRewrittenRowLevelCommand.scala | 34 +
.../v2/RowLevelCommandScanRelationPushDown.scala | 95 +
.../datasources/v2/SetIdentifierFieldsExec.scala | 53 +
.../v2/SetWriteDistributionAndOrderingExec.scala | 78 +
.../execution/datasources/v2/WriteDeltaExec.scala | 238 +++
.../RowLevelCommandDynamicPruning.scala | 171 ++
.../apache/iceberg/spark/extensions/Employee.java | 68 +
.../spark/extensions/SparkExtensionsTestBase.java | 66 +
.../SparkRowLevelOperationsTestBase.java | 243 +++
.../spark/extensions/TestAddFilesProcedure.java | 903 +++++++++
.../extensions/TestAlterTablePartitionFields.java | 415 ++++
.../spark/extensions/TestAlterTableSchema.java | 142 ++
.../spark/extensions/TestAncestorsOfProcedure.java | 163 ++
.../spark/extensions/TestCallStatementParser.java | 196 ++
.../TestCherrypickSnapshotProcedure.java | 180 ++
.../spark/extensions/TestConflictValidation.java | 411 ++++
.../spark/extensions/TestCopyOnWriteDelete.java | 37 +
.../spark/extensions/TestCopyOnWriteMerge.java | 37 +
.../spark/extensions/TestCopyOnWriteUpdate.java | 37 +
.../iceberg/spark/extensions/TestDelete.java | 876 +++++++++
.../extensions/TestExpireSnapshotsProcedure.java | 350 ++++
.../spark/extensions/TestIcebergExpressions.java | 70 +
.../apache/iceberg/spark/extensions/TestMerge.java | 1813 ++++++++++++++++++
.../spark/extensions/TestMergeOnReadDelete.java | 105 +
.../spark/extensions/TestMergeOnReadMerge.java | 40 +
.../spark/extensions/TestMergeOnReadUpdate.java | 40 +
.../spark/extensions/TestMetadataTables.java | 363 ++++
.../extensions/TestMigrateTableProcedure.java | 151 ++
.../extensions/TestRegisterTableProcedure.java | 89 +
.../extensions/TestRemoveOrphanFilesProcedure.java | 403 ++++
.../TestRequiredDistributionAndOrdering.java | 280 +++
.../extensions/TestRewriteDataFilesProcedure.java | 349 ++++
.../extensions/TestRewriteManifestsProcedure.java | 174 ++
.../TestRollbackToSnapshotProcedure.java | 260 +++
.../TestRollbackToTimestampProcedure.java | 268 +++
.../TestSetCurrentSnapshotProcedure.java | 221 +++
.../TestSetWriteDistributionAndOrdering.java | 282 +++
.../extensions/TestSnapshotTableProcedure.java | 191 ++
.../iceberg/spark/extensions/TestUpdate.java | 1001 ++++++++++
spark/v3.2/spark-runtime/LICENSE | 629 ++++++
spark/v3.2/spark-runtime/NOTICE | 508 +++++
.../java/org/apache/iceberg/spark/SmokeTest.java | 160 ++
.../apache/iceberg/spark/SparkBenchmarkUtil.java | 58 +
.../action/IcebergSortCompactionBenchmark.java | 378 ++++
.../iceberg/spark/action/RandomGeneratingUDF.java | 47 +
.../SparkParquetReadersFlatDataBenchmark.java | 216 +++
.../SparkParquetReadersNestedDataBenchmark.java | 216 +++
.../SparkParquetWritersFlatDataBenchmark.java | 125 ++
.../SparkParquetWritersNestedDataBenchmark.java | 124 ++
.../org/apache/iceberg/spark/source/Action.java | 25 +
.../spark/source/IcebergSourceBenchmark.java | 195 ++
.../spark/source/IcebergSourceDeleteBenchmark.java | 287 +++
.../source/IcebergSourceFlatDataBenchmark.java | 59 +
.../source/IcebergSourceNestedDataBenchmark.java | 58 +
.../IcebergSourceNestedListDataBenchmark.java | 56 +
.../iceberg/spark/source/WritersBenchmark.java | 349 ++++
.../spark/source/avro/AvroWritersBenchmark.java | 41 +
.../IcebergSourceFlatAvroDataReadBenchmark.java | 132 ++
.../IcebergSourceNestedAvroDataReadBenchmark.java | 132 ++
.../orc/IcebergSourceFlatORCDataBenchmark.java | 67 +
.../orc/IcebergSourceFlatORCDataReadBenchmark.java | 184 ++
...ebergSourceNestedListORCDataWriteBenchmark.java | 99 +
.../IcebergSourceNestedORCDataReadBenchmark.java | 161 ++
...cebergSourceFlatParquetDataFilterBenchmark.java | 122 ++
.../IcebergSourceFlatParquetDataReadBenchmark.java | 153 ++
...IcebergSourceFlatParquetDataWriteBenchmark.java | 90 +
...gSourceNestedListParquetDataWriteBenchmark.java | 89 +
...bergSourceNestedParquetDataFilterBenchmark.java | 121 ++
...cebergSourceNestedParquetDataReadBenchmark.java | 154 ++
...ebergSourceNestedParquetDataWriteBenchmark.java | 90 +
.../IcebergSourceParquetEqDeleteBenchmark.java | 60 +
...ebergSourceParquetMultiDeleteFileBenchmark.java | 60 +
.../IcebergSourceParquetPosDeleteBenchmark.java | 63 +
...gSourceParquetWithUnrelatedDeleteBenchmark.java | 62 +
.../source/parquet/ParquetWritersBenchmark.java | 41 +
...dDictionaryEncodedFlatParquetDataBenchmark.java | 139 ++
.../VectorizedReadFlatParquetDataBenchmark.java | 304 +++
.../java/org/apache/iceberg/spark/BaseCatalog.java | 49 +
.../org/apache/iceberg/spark/CommitMetadata.java | 61 +
.../iceberg/spark/FileRewriteCoordinator.java | 94 +
.../iceberg/spark/FileScanTaskSetManager.java | 78 +
.../org/apache/iceberg/spark/IcebergSpark.java | 48 +
.../org/apache/iceberg/spark/JobGroupInfo.java | 48 +
.../org/apache/iceberg/spark/JobGroupUtils.java | 46 +
.../org/apache/iceberg/spark/PathIdentifier.java | 57 +
.../iceberg/spark/PruneColumnsWithReordering.java | 258 +++
.../spark/PruneColumnsWithoutReordering.java | 226 +++
.../apache/iceberg/spark/RollbackStagedTable.java | 137 ++
.../org/apache/iceberg/spark/SortOrderToSpark.java | 92 +
.../java/org/apache/iceberg/spark/Spark3Util.java | 894 +++++++++
.../org/apache/iceberg/spark/SparkCatalog.java | 629 ++++++
.../org/apache/iceberg/spark/SparkConfParser.java | 203 ++
.../org/apache/iceberg/spark/SparkDataFile.java | 203 ++
.../spark/SparkDistributionAndOrderingUtil.java | 224 +++
.../apache/iceberg/spark/SparkExceptionUtil.java | 66 +
.../org/apache/iceberg/spark/SparkFilters.java | 271 +++
.../iceberg/spark/SparkFixupTimestampType.java | 57 +
.../org/apache/iceberg/spark/SparkFixupTypes.java | 64 +
.../org/apache/iceberg/spark/SparkReadConf.java | 228 +++
.../org/apache/iceberg/spark/SparkReadOptions.java | 76 +
.../apache/iceberg/spark/SparkSQLProperties.java | 46 +
.../org/apache/iceberg/spark/SparkSchemaUtil.java | 347 ++++
.../apache/iceberg/spark/SparkSessionCatalog.java | 338 ++++
.../org/apache/iceberg/spark/SparkStructLike.java | 55 +
.../org/apache/iceberg/spark/SparkTableUtil.java | 673 +++++++
.../org/apache/iceberg/spark/SparkTypeToType.java | 163 ++
.../org/apache/iceberg/spark/SparkTypeVisitor.java | 83 +
.../java/org/apache/iceberg/spark/SparkUtil.java | 253 +++
.../apache/iceberg/spark/SparkValueConverter.java | 125 ++
.../org/apache/iceberg/spark/SparkWriteConf.java | 269 +++
.../apache/iceberg/spark/SparkWriteOptions.java | 81 +
.../org/apache/iceberg/spark/TypeToSparkType.java | 139 ++
.../actions/BaseDeleteOrphanFilesSparkAction.java | 364 ++++
.../BaseDeleteReachableFilesSparkAction.java | 183 ++
.../actions/BaseExpireSnapshotsSparkAction.java | 276 +++
.../spark/actions/BaseMigrateTableSparkAction.java | 214 +++
.../actions/BaseRewriteDataFilesSparkAction.java | 484 +++++
.../actions/BaseRewriteManifestsSparkAction.java | 370 ++++
.../actions/BaseSnapshotTableSparkAction.java | 204 ++
.../actions/BaseSnapshotUpdateSparkAction.java | 46 +
.../iceberg/spark/actions/BaseSparkAction.java | 231 +++
.../actions/BaseTableCreationSparkAction.java | 168 ++
.../iceberg/spark/actions/ManifestFileBean.java | 153 ++
.../apache/iceberg/spark/actions/SparkActions.java | 104 +
.../spark/actions/SparkBinPackStrategy.java | 104 +
.../iceberg/spark/actions/SparkSortStrategy.java | 170 ++
.../iceberg/spark/actions/SparkZOrderStrategy.java | 219 +++
.../iceberg/spark/actions/SparkZOrderUDF.java | 274 +++
.../spark/data/AvroWithSparkSchemaVisitor.java | 74 +
.../spark/data/ParquetWithSparkSchemaVisitor.java | 208 ++
.../apache/iceberg/spark/data/SparkAvroReader.java | 167 ++
.../apache/iceberg/spark/data/SparkAvroWriter.java | 156 ++
.../apache/iceberg/spark/data/SparkOrcReader.java | 123 ++
.../iceberg/spark/data/SparkOrcValueReaders.java | 232 +++
.../iceberg/spark/data/SparkOrcValueWriters.java | 201 ++
.../apache/iceberg/spark/data/SparkOrcWriter.java | 222 +++
.../iceberg/spark/data/SparkParquetReaders.java | 755 ++++++++
.../iceberg/spark/data/SparkParquetWriters.java | 423 ++++
.../iceberg/spark/data/SparkValueReaders.java | 288 +++
.../iceberg/spark/data/SparkValueWriters.java | 252 +++
.../vectorized/ArrowVectorAccessorFactory.java | 122 ++
.../data/vectorized/ArrowVectorAccessors.java | 40 +
.../data/vectorized/ColumnVectorWithFilter.java | 105 +
.../spark/data/vectorized/ColumnarBatchReader.java | 215 +++
.../data/vectorized/ConstantColumnVector.java | 124 ++
.../data/vectorized/IcebergArrowColumnVector.java | 165 ++
.../data/vectorized/RowPositionColumnVector.java | 122 ++
.../data/vectorized/VectorizedSparkOrcReaders.java | 427 +++++
.../vectorized/VectorizedSparkParquetReaders.java | 92 +
.../spark/procedures/AddFilesProcedure.java | 243 +++
.../spark/procedures/AncestorsOfProcedure.java | 107 ++
.../iceberg/spark/procedures/BaseProcedure.java | 201 ++
.../procedures/CherrypickSnapshotProcedure.java | 97 +
.../spark/procedures/ExpireSnapshotsProcedure.java | 135 ++
.../spark/procedures/MigrateTableProcedure.java | 93 +
.../spark/procedures/RegisterTableProcedure.java | 104 +
.../procedures/RemoveOrphanFilesProcedure.java | 161 ++
.../procedures/RewriteDataFilesProcedure.java | 186 ++
.../procedures/RewriteManifestsProcedure.java | 108 ++
.../procedures/RollbackToSnapshotProcedure.java | 96 +
.../procedures/RollbackToTimestampProcedure.java | 100 +
.../procedures/SetCurrentSnapshotProcedure.java | 97 +
.../spark/procedures/SnapshotTableProcedure.java | 107 ++
.../iceberg/spark/procedures/SparkProcedures.java | 64 +
.../iceberg/spark/source/BaseDataReader.java | 203 ++
.../iceberg/spark/source/BatchDataReader.java | 158 ++
.../spark/source/EqualityDeleteRowReader.java | 54 +
.../iceberg/spark/source/HasIcebergCatalog.java | 31 +
.../apache/iceberg/spark/source/IcebergSource.java | 189 ++
.../iceberg/spark/source/InternalRowWrapper.java | 96 +
.../apache/iceberg/spark/source/RowDataReader.java | 204 ++
.../iceberg/spark/source/RowDataRewriter.java | 153 ++
.../iceberg/spark/source/SparkAppenderFactory.java | 295 +++
.../apache/iceberg/spark/source/SparkBatch.java | 117 ++
.../iceberg/spark/source/SparkBatchQueryScan.java | 275 +++
.../spark/source/SparkCopyOnWriteOperation.java | 103 +
.../iceberg/spark/source/SparkCopyOnWriteScan.java | 179 ++
.../spark/source/SparkFileWriterFactory.java | 245 +++
.../iceberg/spark/source/SparkFilesScan.java | 103 +
.../spark/source/SparkFilesScanBuilder.java | 45 +
.../iceberg/spark/source/SparkMetadataColumn.java | 51 +
.../spark/source/SparkMicroBatchStream.java | 309 +++
.../spark/source/SparkPartitionedFanoutWriter.java | 51 +
.../spark/source/SparkPartitionedWriter.java | 51 +
.../spark/source/SparkPositionDeltaOperation.java | 104 +
.../spark/source/SparkPositionDeltaWrite.java | 668 +++++++
.../source/SparkPositionDeltaWriteBuilder.java | 123 ++
.../source/SparkRowLevelOperationBuilder.java | 112 ++
.../org/apache/iceberg/spark/source/SparkScan.java | 257 +++
.../iceberg/spark/source/SparkScanBuilder.java | 295 +++
.../apache/iceberg/spark/source/SparkTable.java | 339 ++++
.../apache/iceberg/spark/source/SparkWrite.java | 740 +++++++
.../iceberg/spark/source/SparkWriteBuilder.java | 254 +++
.../iceberg/spark/source/StagedSparkTable.java | 42 +
.../org/apache/iceberg/spark/source/Stats.java | 43 +
.../iceberg/spark/source/StreamingOffset.java | 154 ++
.../iceberg/spark/source/StructInternalRow.java | 336 ++++
.../analysis/NoSuchProcedureException.java | 31 +
.../sql/connector/iceberg/catalog/Procedure.java | 60 +
.../iceberg/catalog/ProcedureCatalog.java | 41 +
.../iceberg/catalog/ProcedureParameter.java | 65 +
.../iceberg/catalog/ProcedureParameterImpl.java | 77 +
.../catalog/SupportsRowLevelOperations.java | 39 +
.../connector/iceberg/write/DeltaBatchWrite.java | 31 +
.../sql/connector/iceberg/write/DeltaWrite.java | 33 +
.../connector/iceberg/write/DeltaWriteBuilder.java | 35 +
.../sql/connector/iceberg/write/DeltaWriter.java | 60 +
.../iceberg/write/DeltaWriterFactory.java | 31 +
.../iceberg/write/ExtendedLogicalWriteInfo.java | 38 +
.../connector/iceberg/write/RowLevelOperation.java | 89 +
.../iceberg/write/RowLevelOperationBuilder.java | 30 +
.../iceberg/write/RowLevelOperationInfo.java | 38 +
.../sql/connector/iceberg/write/SupportsDelta.java | 36 +
...org.apache.spark.sql.sources.DataSourceRegister | 20 +
.../expressions/TransformExpressions.scala | 159 ++
.../logical/SetWriteDistributionAndOrdering.scala | 44 +
.../plans/logical/SortOrderParserUtil.scala | 40 +
.../utils/DistributionAndOrderingUtils.scala | 189 ++
.../spark/sql/catalyst/utils/PlanUtils.scala | 42 +
.../datasources/SparkExpressionConverter.scala | 51 +
.../test/java/org/apache/iceberg/KryoHelpers.java | 52 +
.../java/org/apache/iceberg/TaskCheckHelper.java | 93 +
.../apache/iceberg/TestDataFileSerialization.java | 170 ++
.../apache/iceberg/TestFileIOSerialization.java | 113 ++
.../TestHadoopMetricsContextSerialization.java | 56 +
.../iceberg/TestManifestFileSerialization.java | 194 ++
.../apache/iceberg/TestScanTaskSerialization.java | 199 ++
.../org/apache/iceberg/TestTableSerialization.java | 102 +
.../apache/iceberg/spark/SparkCatalogConfig.java | 61 +
.../apache/iceberg/spark/SparkCatalogTestBase.java | 59 +
.../org/apache/iceberg/spark/SparkTestBase.java | 254 +++
.../iceberg/spark/SparkTestBaseWithCatalog.java | 95 +
.../iceberg/spark/TestFileRewriteCoordinator.java | 252 +++
.../org/apache/iceberg/spark/TestSpark3Util.java | 118 ++
.../iceberg/spark/TestSparkCatalogOperations.java | 98 +
.../TestSparkDistributionAndOrderingUtil.java | 2022 ++++++++++++++++++++
.../org/apache/iceberg/spark/TestSparkFilters.java | 151 ++
.../apache/iceberg/spark/TestSparkSchemaUtil.java | 82 +
.../iceberg/spark/TestSparkSessionCatalog.java | 66 +
.../apache/iceberg/spark/TestSparkTableUtil.java | 87 +
.../iceberg/spark/TestSparkValueConverter.java | 90 +
.../iceberg/spark/actions/TestCreateActions.java | 896 +++++++++
.../actions/TestDeleteReachableFilesAction.java | 353 ++++
.../spark/actions/TestExpireSnapshotsAction.java | 1153 +++++++++++
.../spark/actions/TestRemoveOrphanFilesAction.java | 958 ++++++++++
.../actions/TestRemoveOrphanFilesAction3.java | 175 ++
.../spark/actions/TestRewriteDataFilesAction.java | 1564 +++++++++++++++
.../spark/actions/TestRewriteManifestsAction.java | 493 +++++
.../apache/iceberg/spark/data/AvroDataTest.java | 240 +++
.../apache/iceberg/spark/data/GenericsHelpers.java | 311 +++
.../org/apache/iceberg/spark/data/RandomData.java | 363 ++++
.../org/apache/iceberg/spark/data/TestHelpers.java | 736 +++++++
.../apache/iceberg/spark/data/TestOrcWrite.java | 61 +
.../iceberg/spark/data/TestParquetAvroReader.java | 220 +++
.../iceberg/spark/data/TestParquetAvroWriter.java | 107 ++
.../iceberg/spark/data/TestSparkAvroEnums.java | 95 +
.../iceberg/spark/data/TestSparkAvroReader.java | 66 +
.../iceberg/spark/data/TestSparkDateTimes.java | 75 +
.../data/TestSparkOrcReadMetadataColumns.java | 209 ++
.../iceberg/spark/data/TestSparkOrcReader.java | 107 ++
.../data/TestSparkParquetReadMetadataColumns.java | 297 +++
.../iceberg/spark/data/TestSparkParquetReader.java | 203 ++
.../iceberg/spark/data/TestSparkParquetWriter.java | 100 +
.../spark/data/TestSparkRecordOrcReaderWriter.java | 148 ++
...estParquetDictionaryEncodedVectorizedReads.java | 92 +
...naryFallbackToPlainEncodingVectorizedReads.java | 74 +
.../vectorized/TestParquetVectorizedReads.java | 316 +++
.../spark/source/FilePathLastModifiedRecord.java | 78 +
.../apache/iceberg/spark/source/LogMessage.java | 120 ++
.../apache/iceberg/spark/source/ManualSource.java | 74 +
.../apache/iceberg/spark/source/SimpleRecord.java | 80 +
.../apache/iceberg/spark/source/TestAvroScan.java | 115 ++
.../spark/source/TestDataFrameWriterV2.java | 161 ++
.../iceberg/spark/source/TestDataFrameWrites.java | 385 ++++
.../spark/source/TestDataSourceOptions.java | 448 +++++
.../iceberg/spark/source/TestFilteredScan.java | 620 ++++++
.../spark/source/TestForwardCompatibility.java | 217 +++
.../iceberg/spark/source/TestIcebergSource.java | 44 +
.../source/TestIcebergSourceHadoopTables.java | 68 +
.../spark/source/TestIcebergSourceHiveTables.java | 75 +
.../spark/source/TestIcebergSourceTablesBase.java | 1544 +++++++++++++++
.../iceberg/spark/source/TestIcebergSpark.java | 225 +++
.../spark/source/TestIdentityPartitionData.java | 185 ++
.../spark/source/TestInternalRowWrapper.java | 78 +
.../TestMetadataTablesWithPartitionEvolution.java | 806 ++++++++
.../iceberg/spark/source/TestParquetScan.java | 139 ++
.../iceberg/spark/source/TestPartitionPruning.java | 414 ++++
.../iceberg/spark/source/TestPartitionValues.java | 435 +++++
.../iceberg/spark/source/TestPathIdentifier.java | 88 +
.../iceberg/spark/source/TestReadProjection.java | 578 ++++++
.../TestRequiredDistributionAndOrdering.java | 316 +++
.../iceberg/spark/source/TestRuntimeFiltering.java | 415 ++++
.../spark/source/TestSnapshotSelection.java | 238 +++
.../spark/source/TestSparkAppenderFactory.java | 70 +
.../spark/source/TestSparkBaseDataReader.java | 278 +++
.../iceberg/spark/source/TestSparkCatalog.java | 64 +
.../source/TestSparkCatalogCacheExpiration.java | 140 ++
.../source/TestSparkCatalogHadoopOverrides.java | 133 ++
.../iceberg/spark/source/TestSparkDataFile.java | 216 +++
.../iceberg/spark/source/TestSparkDataWrite.java | 658 +++++++
.../spark/source/TestSparkFileWriterFactory.java | 73 +
.../iceberg/spark/source/TestSparkFilesScan.java | 121 ++
.../spark/source/TestSparkMergingMetrics.java | 68 +
.../spark/source/TestSparkMetadataColumns.java | 288 +++
.../spark/source/TestSparkPartitioningWriters.java | 73 +
.../source/TestSparkPositionDeltaWriters.java | 73 +
.../spark/source/TestSparkReadProjection.java | 243 +++
.../spark/source/TestSparkReaderDeletes.java | 481 +++++
.../spark/source/TestSparkRollingFileWriters.java | 59 +
.../iceberg/spark/source/TestSparkTable.java | 61 +
.../spark/source/TestSparkWriterMetrics.java | 68 +
.../iceberg/spark/source/TestStreamingOffset.java | 54 +
.../spark/source/TestStructuredStreaming.java | 305 +++
.../spark/source/TestStructuredStreamingRead3.java | 547 ++++++
.../apache/iceberg/spark/source/TestTables.java | 206 ++
.../spark/source/TestTimestampWithoutZone.java | 290 +++
.../spark/source/TestWriteMetricsConfig.java | 306 +++
.../iceberg/spark/source/ThreeColumnRecord.java | 89 +
.../apache/iceberg/spark/sql/TestAlterTable.java | 276 +++
.../apache/iceberg/spark/sql/TestCreateTable.java | 293 +++
.../iceberg/spark/sql/TestCreateTableAsSelect.java | 363 ++++
.../apache/iceberg/spark/sql/TestDeleteFrom.java | 161 ++
.../apache/iceberg/spark/sql/TestDropTable.java | 140 ++
.../apache/iceberg/spark/sql/TestNamespaceSQL.java | 221 +++
.../iceberg/spark/sql/TestPartitionedWrites.java | 179 ++
.../spark/sql/TestPartitionedWritesAsSelect.java | 112 ++
.../apache/iceberg/spark/sql/TestRefreshTable.java | 76 +
.../org/apache/iceberg/spark/sql/TestSelect.java | 218 +++
.../spark/sql/TestTimestampWithoutZone.java | 196 ++
.../iceberg/spark/sql/TestUnpartitionedWrites.java | 174 ++
388 files changed, 75898 insertions(+)
diff --git a/spark/v3.2/build.gradle b/spark/v3.2/build.gradle
new file mode 100644
index 000000000..ae76a0f91
--- /dev/null
+++ b/spark/v3.2/build.gradle
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+String sparkVersion = '3.2.1'
+String sparkMajorVersion = '3.2'
+String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
+
+def sparkProjects = [
+ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}"),
+ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}"),
+ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}"),
+]
+
+configure(sparkProjects) {
+ configurations {
+ all {
+ resolutionStrategy {
+ force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:2.12.3"
+ force 'com.fasterxml.jackson.module:jackson-module-paranamer:2.12.3'
+ }
+ }
+ }
+}
+
+project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
+ apply plugin: 'scala'
+ apply plugin: 'com.github.alisiikh.scalastyle'
+
+ sourceSets {
+ main {
+ scala.srcDirs = ['src/main/scala', 'src/main/java']
+ java.srcDirs = []
+ }
+ }
+
+ dependencies {
+ implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+ api project(':iceberg-api')
+ implementation project(':iceberg-common')
+ implementation project(':iceberg-core')
+ implementation project(':iceberg-data')
+ implementation project(':iceberg-orc')
+ implementation project(':iceberg-parquet')
+ implementation project(':iceberg-arrow')
+ implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}")
+
+ compileOnly "com.google.errorprone:error_prone_annotations"
+ compileOnly "org.apache.avro:avro"
+ compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
+ // to make sure io.netty.buffer only comes from project(':iceberg-arrow')
+ exclude group: 'io.netty', module: 'netty-buffer'
+ exclude group: 'org.roaringbitmap'
+ }
+
+ implementation("org.apache.parquet:parquet-column")
+ implementation("org.apache.parquet:parquet-hadoop")
+
+ implementation("org.apache.orc:orc-core::nohive") {
+ exclude group: 'org.apache.hadoop'
+ exclude group: 'commons-lang'
+ // These artifacts are shaded and included in the orc-core fat jar
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.hive', module: 'hive-storage-api'
+ }
+
+ implementation("org.apache.arrow:arrow-vector") {
+ exclude group: 'io.netty', module: 'netty-buffer'
+ exclude group: 'io.netty', module: 'netty-common'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+
+ testImplementation("org.apache.hadoop:hadoop-minicluster") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ // to make sure io.netty.buffer only comes from project(':iceberg-arrow')
+ exclude group: 'io.netty', module: 'netty-buffer'
+ }
+ testImplementation project(path: ':iceberg-hive-metastore')
+ testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
+ testImplementation "org.xerial:sqlite-jdbc"
+ }
+
+ tasks.withType(Test) {
+ // For vectorized reads
+ // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds
+ systemProperty("arrow.enable_unsafe_memory_access", "true")
+ // Disable expensive null check for every get(index) call.
+ // Iceberg manages nullability checks itself instead of relying on arrow.
+ systemProperty("arrow.enable_null_check_for_get", "false")
+
+ // Vectorized reads need more memory
+ maxHeapSize '2560m'
+ }
+}
+
+project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") {
+ apply plugin: 'java-library'
+ apply plugin: 'scala'
+ apply plugin: 'com.github.alisiikh.scalastyle'
+ apply plugin: 'antlr'
+
+ configurations {
+ /*
+ The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This
+ bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate
+ runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with
+ the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add
+ the runtime dependency back to only the runtime configuration manually.
+ */
+ implementation {
+ extendsFrom = extendsFrom.findAll { it != configurations.antlr }
+ }
+ }
+
+ dependencies {
+ implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}")
+
+ compileOnly "org.scala-lang:scala-library:${scalaVersion}"
+ compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+ compileOnly project(':iceberg-api')
+ compileOnly project(':iceberg-core')
+ compileOnly project(':iceberg-common')
+ compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
+ compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
+ // to make sure io.netty.buffer only comes from project(':iceberg-arrow')
+ exclude group: 'io.netty', module: 'netty-buffer'
+ exclude group: 'org.roaringbitmap'
+ }
+
+ testImplementation project(path: ':iceberg-hive-metastore')
+ testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+
+ testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+
+ testImplementation "org.apache.avro:avro"
+
+ // Required because we remove antlr plugin dependencies from the compile configuration, see note above
+ runtimeOnly "org.antlr:antlr4-runtime:4.8"
+ antlr "org.antlr:antlr4:4.8"
+ }
+
+ generateGrammarSource {
+ maxHeapSize = "64m"
+ arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions']
+ }
+}
+
+project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}") {
+ apply plugin: 'com.github.johnrengelman.shadow'
+
+ tasks.jar.dependsOn tasks.shadowJar
+
+ sourceSets {
+ integration {
+ java.srcDir "$projectDir/src/integration/java"
+ resources.srcDir "$projectDir/src/integration/resources"
+ }
+ }
+
+ configurations {
+ implementation {
+ exclude group: 'org.apache.spark'
+ // included in Spark
+ exclude group: 'org.slf4j'
+ exclude group: 'org.apache.commons'
+ exclude group: 'commons-pool'
+ exclude group: 'commons-codec'
+ exclude group: 'org.xerial.snappy'
+ exclude group: 'javax.xml.bind'
+ exclude group: 'javax.annotation'
+ exclude group: 'com.github.luben'
+ exclude group: 'com.ibm.icu'
+ exclude group: 'org.glassfish'
+ exclude group: 'org.abego.treelayout'
+ exclude group: 'org.antlr'
+ }
+ }
+
+ dependencies {
+ api project(':iceberg-api')
+ implementation project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
+ implementation project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
+ implementation project(':iceberg-aws')
+ implementation(project(':iceberg-aliyun')) {
+ exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
+ exclude group: 'org.apache.httpcomponents', module: 'httpclient'
+ exclude group: 'commons-logging', module: 'commons-logging'
+ }
+ implementation project(':iceberg-hive-metastore')
+ implementation(project(':iceberg-nessie')) {
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+
+ integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}"
+ integrationImplementation 'org.junit.vintage:junit-vintage-engine'
+ integrationImplementation 'org.slf4j:slf4j-simple'
+ integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+ integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+ // Not allowed on our classpath, only the runtime jar is allowed
+ integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
+ integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
+ integrationCompileOnly project(':iceberg-api')
+ }
+
+ shadowJar {
+ configurations = [project.configurations.runtimeClasspath]
+
+ zip64 true
+
+ // include the LICENSE and NOTICE files for the shaded Jar
+ from(projectDir) {
+ include 'LICENSE'
+ include 'NOTICE'
+ }
+
+ // Relocate dependencies to avoid conflicts
+ relocate 'com.google', 'org.apache.iceberg.shaded.com.google'
+ relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
+ relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
+ relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
+ relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
+ relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
+ relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
+ relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
+ relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
+ relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
+ relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
+ // relocate Arrow and related deps to shade Iceberg specific version
+ relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer'
+ relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow'
+ relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch'
+ relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
+ relocate 'org.roaringbitmap', 'org.apache.iceberg.shaded.org.roaringbitmap'
+
+ classifier null
+ }
+
+ task integrationTest(type: Test) {
+ description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
+ group = "verification"
+ testClassesDirs = sourceSets.integration.output.classesDirs
+ classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
+ inputs.file(shadowJar.archiveFile.get().asFile.path)
+ }
+ integrationTest.dependsOn shadowJar
+
+ jar {
+ enabled = false
+ }
+}
+
diff --git a/spark/v3.2/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 b/spark/v3.2/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
new file mode 100644
index 000000000..d0b228df0
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * This file is an adaptation of Presto's and Spark's grammar files.
+ */
+
+grammar IcebergSqlExtensions;
+
+@lexer::members {
+ /**
+ * Verify whether current token is a valid decimal token (which contains dot).
+ * Returns true if the character that follows the token is not a digit or letter or underscore.
+ *
+ * For example:
+ * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'.
+ * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'.
+ * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'.
+ * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed
+ * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+'
+ * which is not a digit or letter or underscore.
+ */
+ public boolean isValidDecimal() {
+ int nextChar = _input.LA(1);
+ if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' ||
+ nextChar == '_') {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * This method will be called when we see '/*' and try to match it as a bracketed comment.
+ * If the next character is '+', it should be parsed as hint later, and we cannot match
+ * it as a bracketed comment.
+ *
+ * Returns true if the next character is '+'.
+ */
+ public boolean isHint() {
+ int nextChar = _input.LA(1);
+ if (nextChar == '+') {
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
+
+singleStatement
+ : statement EOF
+ ;
+
+statement
+ : CALL multipartIdentifier '(' (callArgument (',' callArgument)*)? ')' #call
+ | ALTER TABLE multipartIdentifier ADD PARTITION FIELD transform (AS name=identifier)? #addPartitionField
+ | ALTER TABLE multipartIdentifier DROP PARTITION FIELD transform #dropPartitionField
+ | ALTER TABLE multipartIdentifier REPLACE PARTITION FIELD transform WITH transform (AS name=identifier)? #replacePartitionField
+ | ALTER TABLE multipartIdentifier WRITE writeSpec #setWriteDistributionAndOrdering
+ | ALTER TABLE multipartIdentifier SET IDENTIFIER_KW FIELDS fieldList #setIdentifierFields
+ | ALTER TABLE multipartIdentifier DROP IDENTIFIER_KW FIELDS fieldList #dropIdentifierFields
+ ;
+
+writeSpec
+ : (writeDistributionSpec | writeOrderingSpec)*
+ ;
+
+writeDistributionSpec
+ : DISTRIBUTED BY PARTITION
+ ;
+
+writeOrderingSpec
+ : LOCALLY? ORDERED BY order
+ | UNORDERED
+ ;
+
+callArgument
+ : expression #positionalArgument
+ | identifier '=>' expression #namedArgument
+ ;
+
+order
+ : fields+=orderField (',' fields+=orderField)*
+ | '(' fields+=orderField (',' fields+=orderField)* ')'
+ ;
+
+orderField
+ : transform direction=(ASC | DESC)? (NULLS nullOrder=(FIRST | LAST))?
+ ;
+
+transform
+ : multipartIdentifier #identityTransform
+ | transformName=identifier
+ '(' arguments+=transformArgument (',' arguments+=transformArgument)* ')' #applyTransform
+ ;
+
+transformArgument
+ : multipartIdentifier
+ | constant
+ ;
+
+expression
+ : constant
+ | stringMap
+ ;
+
+constant
+ : number #numericLiteral
+ | booleanValue #booleanLiteral
+ | STRING+ #stringLiteral
+ | identifier STRING #typeConstructor
+ ;
+
+stringMap
+ : MAP '(' constant (',' constant)* ')'
+ ;
+
+booleanValue
+ : TRUE | FALSE
+ ;
+
+number
+ : MINUS? EXPONENT_VALUE #exponentLiteral
+ | MINUS? DECIMAL_VALUE #decimalLiteral
+ | MINUS? INTEGER_VALUE #integerLiteral
+ | MINUS? BIGINT_LITERAL #bigIntLiteral
+ | MINUS? SMALLINT_LITERAL #smallIntLiteral
+ | MINUS? TINYINT_LITERAL #tinyIntLiteral
+ | MINUS? DOUBLE_LITERAL #doubleLiteral
+ | MINUS? FLOAT_LITERAL #floatLiteral
+ | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral
+ ;
+
+multipartIdentifier
+ : parts+=identifier ('.' parts+=identifier)*
+ ;
+
+identifier
+ : IDENTIFIER #unquotedIdentifier
+ | quotedIdentifier #quotedIdentifierAlternative
+ | nonReserved #unquotedIdentifier
+ ;
+
+quotedIdentifier
+ : BACKQUOTED_IDENTIFIER
+ ;
+
+fieldList
+ : fields+=multipartIdentifier (',' fields+=multipartIdentifier)*
+ ;
+
+nonReserved
+ : ADD | ALTER | AS | ASC | BY | CALL | DESC | DROP | FIELD | FIRST | LAST | NULLS | ORDERED | PARTITION | TABLE | WRITE
+ | DISTRIBUTED | LOCALLY | UNORDERED | REPLACE | WITH | IDENTIFIER_KW | FIELDS | SET
+ | TRUE | FALSE
+ | MAP
+ ;
+
+ADD: 'ADD';
+ALTER: 'ALTER';
+AS: 'AS';
+ASC: 'ASC';
+BY: 'BY';
+CALL: 'CALL';
+DESC: 'DESC';
+DISTRIBUTED: 'DISTRIBUTED';
+DROP: 'DROP';
+FIELD: 'FIELD';
+FIELDS: 'FIELDS';
+FIRST: 'FIRST';
+LAST: 'LAST';
+LOCALLY: 'LOCALLY';
+NULLS: 'NULLS';
+ORDERED: 'ORDERED';
+PARTITION: 'PARTITION';
+REPLACE: 'REPLACE';
+IDENTIFIER_KW: 'IDENTIFIER';
+SET: 'SET';
+TABLE: 'TABLE';
+UNORDERED: 'UNORDERED';
+WITH: 'WITH';
+WRITE: 'WRITE';
+
+TRUE: 'TRUE';
+FALSE: 'FALSE';
+
+MAP: 'MAP';
+
+PLUS: '+';
+MINUS: '-';
+
+STRING
+ : '\'' ( ~('\''|'\\') | ('\\' .) )* '\''
+ | '"' ( ~('"'|'\\') | ('\\' .) )* '"'
+ ;
+
+BIGINT_LITERAL
+ : DIGIT+ 'L'
+ ;
+
+SMALLINT_LITERAL
+ : DIGIT+ 'S'
+ ;
+
+TINYINT_LITERAL
+ : DIGIT+ 'Y'
+ ;
+
+INTEGER_VALUE
+ : DIGIT+
+ ;
+
+EXPONENT_VALUE
+ : DIGIT+ EXPONENT
+ | DECIMAL_DIGITS EXPONENT {isValidDecimal()}?
+ ;
+
+DECIMAL_VALUE
+ : DECIMAL_DIGITS {isValidDecimal()}?
+ ;
+
+FLOAT_LITERAL
+ : DIGIT+ EXPONENT? 'F'
+ | DECIMAL_DIGITS EXPONENT? 'F' {isValidDecimal()}?
+ ;
+
+DOUBLE_LITERAL
+ : DIGIT+ EXPONENT? 'D'
+ | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}?
+ ;
+
+BIGDECIMAL_LITERAL
+ : DIGIT+ EXPONENT? 'BD'
+ | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}?
+ ;
+
+IDENTIFIER
+ : (LETTER | DIGIT | '_')+
+ ;
+
+BACKQUOTED_IDENTIFIER
+ : '`' ( ~'`' | '``' )* '`'
+ ;
+
+fragment DECIMAL_DIGITS
+ : DIGIT+ '.' DIGIT*
+ | '.' DIGIT+
+ ;
+
+fragment EXPONENT
+ : 'E' [+-]? DIGIT+
+ ;
+
+fragment DIGIT
+ : [0-9]
+ ;
+
+fragment LETTER
+ : [A-Z]
+ ;
+
+SIMPLE_COMMENT
+ : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
+ ;
+
+BRACKETED_COMMENT
+ : '/*' {!isHint()}? (BRACKETED_COMMENT|.)*? '*/' -> channel(HIDDEN)
+ ;
+
+WS
+ : [ \r\n\t]+ -> channel(HIDDEN)
+ ;
+
+// Catch-all for anything we can't recognize.
+// We use this to be able to ignore and recover all the text
+// when splitting statements with DelimiterLexer
+UNRECOGNIZED
+ : .
+ ;
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
new file mode 100644
index 000000000..455129f2c
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions
+
+import org.apache.spark.sql.SparkSessionExtensions
+import org.apache.spark.sql.catalyst.analysis.AlignedRowLevelIcebergCommandCheck
+import org.apache.spark.sql.catalyst.analysis.AlignRowLevelCommandAssignments
+import org.apache.spark.sql.catalyst.analysis.CheckMergeIntoTableConditions
+import org.apache.spark.sql.catalyst.analysis.MergeIntoIcebergTableResolutionCheck
+import org.apache.spark.sql.catalyst.analysis.ProcedureArgumentCoercion
+import org.apache.spark.sql.catalyst.analysis.ResolveMergeIntoTableReferences
+import org.apache.spark.sql.catalyst.analysis.ResolveProcedures
+import org.apache.spark.sql.catalyst.analysis.RewriteDeleteFromTable
+import org.apache.spark.sql.catalyst.analysis.RewriteMergeIntoTable
+import org.apache.spark.sql.catalyst.analysis.RewriteUpdateTable
+import org.apache.spark.sql.catalyst.optimizer.ExtendedReplaceNullWithFalseInPredicate
+import org.apache.spark.sql.catalyst.optimizer.ExtendedSimplifyConditionalsInPredicate
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser
+import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy
+import org.apache.spark.sql.execution.datasources.v2.ExtendedV2Writes
+import org.apache.spark.sql.execution.datasources.v2.OptimizeMetadataOnlyDeleteFromTable
+import org.apache.spark.sql.execution.datasources.v2.ReplaceRewrittenRowLevelCommand
+import org.apache.spark.sql.execution.datasources.v2.RowLevelCommandScanRelationPushDown
+import org.apache.spark.sql.execution.dynamicpruning.RowLevelCommandDynamicPruning
+
+class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
+
+ override def apply(extensions: SparkSessionExtensions): Unit = {
+ // parser extensions
+ extensions.injectParser { case (_, parser) => new IcebergSparkSqlExtensionsParser(parser) }
+
+ // analyzer extensions
+ extensions.injectResolutionRule { spark => ResolveProcedures(spark) }
+ extensions.injectResolutionRule { spark => ResolveMergeIntoTableReferences(spark) }
+ extensions.injectResolutionRule { _ => CheckMergeIntoTableConditions }
+ extensions.injectResolutionRule { _ => ProcedureArgumentCoercion }
+ extensions.injectResolutionRule { _ => AlignRowLevelCommandAssignments }
+ extensions.injectResolutionRule { _ => RewriteDeleteFromTable }
+ extensions.injectResolutionRule { _ => RewriteUpdateTable }
+ extensions.injectResolutionRule { _ => RewriteMergeIntoTable }
+ extensions.injectCheckRule { _ => MergeIntoIcebergTableResolutionCheck }
+ extensions.injectCheckRule { _ => AlignedRowLevelIcebergCommandCheck }
+
+ // optimizer extensions
+ extensions.injectOptimizerRule { _ => ExtendedSimplifyConditionalsInPredicate }
+ extensions.injectOptimizerRule { _ => ExtendedReplaceNullWithFalseInPredicate }
+ // pre-CBO rules run only once and the order of the rules is important
+ // - metadata deletes have to be attempted immediately after the operator optimization
+ // - dynamic filters should be added before replacing commands with rewrite plans
+ // - scans must be planned before building writes
+ extensions.injectPreCBORule { _ => OptimizeMetadataOnlyDeleteFromTable }
+ extensions.injectPreCBORule { _ => RowLevelCommandScanRelationPushDown }
+ extensions.injectPreCBORule { _ => ExtendedV2Writes }
+ extensions.injectPreCBORule { spark => RowLevelCommandDynamicPruning(spark) }
+ extensions.injectPreCBORule { _ => ReplaceRewrittenRowLevelCommand }
+
+ // planner extensions
+ extensions.injectPlannerStrategy { spark => ExtendedDataSourceV2Strategy(spark) }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala
new file mode 100644
index 000000000..fb654b646
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import org.apache.spark.sql.catalyst.util.ArrayData
+import org.apache.spark.sql.catalyst.util.MapData
+import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types.Decimal
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.unsafe.types.CalendarInterval
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * An InternalRow that projects particular columns from another InternalRow without copying
+ * the underlying data.
+ */
+case class ProjectingInternalRow(schema: StructType, colOrdinals: Seq[Int]) extends InternalRow {
+ assert(schema.size == colOrdinals.size)
+
+ private var row: InternalRow = _
+
+ override def numFields: Int = colOrdinals.size
+
+ def project(row: InternalRow): Unit = {
+ this.row = row
+ }
+
+ override def setNullAt(i: Int): Unit = {
+ throw new UnsupportedOperationException("Cannot modify InternalRowProjection")
+ }
+
+ override def update(i: Int, value: Any): Unit = {
+ throw new UnsupportedOperationException("Cannot modify InternalRowProjection")
+ }
+
+ override def copy(): InternalRow = {
+ val newRow = if (row != null) row.copy() else null
+ val newProjection = ProjectingInternalRow(schema, colOrdinals)
+ newProjection.project(newRow)
+ newProjection
+ }
+
+ override def isNullAt(ordinal: Int): Boolean = {
+ row.isNullAt(colOrdinals(ordinal))
+ }
+
+ override def getBoolean(ordinal: Int): Boolean = {
+ row.getBoolean(colOrdinals(ordinal))
+ }
+
+ override def getByte(ordinal: Int): Byte = {
+ row.getByte(colOrdinals(ordinal))
+ }
+
+ override def getShort(ordinal: Int): Short = {
+ row.getShort(colOrdinals(ordinal))
+ }
+
+ override def getInt(ordinal: Int): Int = {
+ row.getInt(colOrdinals(ordinal))
+ }
+
+ override def getLong(ordinal: Int): Long = {
+ row.getLong(colOrdinals(ordinal))
+ }
+
+ override def getFloat(ordinal: Int): Float = {
+ row.getFloat(colOrdinals(ordinal))
+ }
+
+ override def getDouble(ordinal: Int): Double = {
+ row.getDouble(colOrdinals(ordinal))
+ }
+
+ override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = {
+ row.getDecimal(colOrdinals(ordinal), precision, scale)
+ }
+
+ override def getUTF8String(ordinal: Int): UTF8String = {
+ row.getUTF8String(colOrdinals(ordinal))
+ }
+
+ override def getBinary(ordinal: Int): Array[Byte] = {
+ row.getBinary(colOrdinals(ordinal))
+ }
+
+ override def getInterval(ordinal: Int): CalendarInterval = {
+ row.getInterval(colOrdinals(ordinal))
+ }
+
+ override def getStruct(ordinal: Int, numFields: Int): InternalRow = {
+ row.getStruct(colOrdinals(ordinal), numFields)
+ }
+
+ override def getArray(ordinal: Int): ArrayData = {
+ row.getArray(colOrdinals(ordinal))
+ }
+
+ override def getMap(ordinal: Int): MapData = {
+ row.getMap(colOrdinals(ordinal))
+ }
+
+ override def get(ordinal: Int, dataType: DataType): AnyRef = {
+ row.get(colOrdinals(ordinal), dataType)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala
new file mode 100644
index 000000000..ad416f2a1
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
+import org.apache.spark.sql.catalyst.plans.logical.Assignment
+import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
+import org.apache.spark.sql.catalyst.plans.logical.InsertAction
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
+import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
+import org.apache.spark.sql.catalyst.rules.Rule
+
+/**
+ * A rule that aligns assignments in UPDATE and MERGE operations.
+ *
+ * Note that this rule must be run before rewriting row-level commands.
+ */
+object AlignRowLevelCommandAssignments
+ extends Rule[LogicalPlan] with AssignmentAlignmentSupport {
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case u: UpdateIcebergTable if u.resolved && !u.aligned =>
+ u.copy(assignments = alignAssignments(u.table, u.assignments))
+
+ case m: MergeIntoIcebergTable if m.resolved && !m.aligned =>
+ val alignedMatchedActions = m.matchedActions.map {
+ case u @ UpdateAction(_, assignments) =>
+ u.copy(assignments = alignAssignments(m.targetTable, assignments))
+ case d: DeleteAction =>
+ d
+ case _ =>
+ throw new AnalysisException("Matched actions can only contain UPDATE or DELETE")
+ }
+
+ val alignedNotMatchedActions = m.notMatchedActions.map {
+ case i @ InsertAction(_, assignments) =>
+ // check no nested columns are present
+ val refs = assignments.map(_.key).map(AssignmentUtils.toAssignmentRef)
+ refs.foreach { ref =>
+ if (ref.size > 1) {
+ throw new AnalysisException(
+ "Nested fields are not supported inside INSERT clauses of MERGE operations: " +
+ s"${ref.mkString("`", "`.`", "`")}")
+ }
+ }
+
+ val colNames = refs.map(_.head)
+
+ // check there are no duplicates
+ val duplicateColNames = colNames.groupBy(identity).collect {
+ case (name, matchingNames) if matchingNames.size > 1 => name
+ }
+
+ if (duplicateColNames.nonEmpty) {
+ throw new AnalysisException(
+ s"Duplicate column names inside INSERT clause: ${duplicateColNames.mkString(", ")}")
+ }
+
+ // reorder assignments by the target table column order
+ val assignmentMap = colNames.zip(assignments).toMap
+ i.copy(assignments = alignInsertActionAssignments(m.targetTable, assignmentMap))
+
+ case _ =>
+ throw new AnalysisException("Not matched actions can only contain INSERT")
+ }
+
+ m.copy(matchedActions = alignedMatchedActions, notMatchedActions = alignedNotMatchedActions)
+ }
+
+ private def alignInsertActionAssignments(
+ targetTable: LogicalPlan,
+ assignmentMap: Map[String, Assignment]): Seq[Assignment] = {
+
+ val resolver = conf.resolver
+
+ targetTable.output.map { targetAttr =>
+ val assignment = assignmentMap
+ .find { case (name, _) => resolver(name, targetAttr.name) }
+ .map { case (_, assignment) => assignment }
+
+ if (assignment.isEmpty) {
+ throw new AnalysisException(
+ s"Cannot find column '${targetAttr.name}' of the target table among " +
+ s"the INSERT columns: ${assignmentMap.keys.mkString(", ")}. " +
+ "INSERT clauses must provide values for all columns of the target table.")
+ }
+
+ val key = assignment.get.key
+ val value = castIfNeeded(targetAttr, assignment.get.value, resolver)
+ AssignmentUtils.handleCharVarcharLimits(Assignment(key, value))
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala
new file mode 100644
index 000000000..d915e4f10
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
+
+object AlignedRowLevelIcebergCommandCheck extends (LogicalPlan => Unit) {
+
+ override def apply(plan: LogicalPlan): Unit = {
+ plan foreach {
+ case m: MergeIntoIcebergTable if !m.aligned =>
+ throw new AnalysisException(s"Could not align Iceberg MERGE INTO: $m")
+ case u: UpdateIcebergTable if !u.aligned =>
+ throw new AnalysisException(s"Could not align Iceberg UPDATE: $u")
+ case _ => // OK
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala
new file mode 100644
index 000000000..14115bd3c
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.SQLConfHelper
+import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.expressions.AnsiCast
+import org.apache.spark.sql.catalyst.expressions.AssignmentUtils._
+import org.apache.spark.sql.catalyst.expressions.Cast
+import org.apache.spark.sql.catalyst.expressions.CreateNamedStruct
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.GetStructField
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.NamedExpression
+import org.apache.spark.sql.catalyst.plans.logical.Assignment
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
+import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types.StructField
+import org.apache.spark.sql.types.StructType
+import scala.collection.compat.immutable.ArraySeq
+import scala.collection.mutable
+
+trait AssignmentAlignmentSupport extends CastSupport {
+
+ self: SQLConfHelper =>
+
+ private case class ColumnUpdate(ref: Seq[String], expr: Expression)
+
+ /**
+ * Aligns assignments to match table columns.
+ * <p>
+ * This method processes and reorders given assignments so that each target column gets
+ * an expression it should be set to. If a column does not have a matching assignment,
+ * it will be set to its current value. For example, if one passes a table with columns c1, c2
+ * and an assignment c2 = 1, this method will return c1 = c1, c2 = 1.
+ * <p>
+ * This method also handles updates to nested columns. If there is an assignment to a particular
+ * nested field, this method will construct a new struct with one field updated
+ * preserving other fields that have not been modified. For example, if one passes a table with
+ * columns c1, c2 where c2 is a struct with fields n1 and n2 and an assignment c2.n2 = 1,
+ * this method will return c1 = c1, c2 = struct(c2.n1, 1).
+ *
+ * @param table a target table
+ * @param assignments assignments to align
+ * @return aligned assignments that match table columns
+ */
+ protected def alignAssignments(
+ table: LogicalPlan,
+ assignments: Seq[Assignment]): Seq[Assignment] = {
+
+ val columnUpdates = assignments.map(a => ColumnUpdate(toAssignmentRef(a.key), a.value))
+ val outputExprs = applyUpdates(table.output, columnUpdates)
+ outputExprs.zip(table.output).map {
+ case (expr, attr) => handleCharVarcharLimits(Assignment(attr, expr))
+ }
+ }
+
+ private def applyUpdates(
+ cols: Seq[NamedExpression],
+ updates: Seq[ColumnUpdate],
+ resolver: Resolver = conf.resolver,
+ namePrefix: Seq[String] = Nil): Seq[Expression] = {
+
+ // iterate through columns at the current level and find which column updates match
+ cols.map { col =>
+ // find matches for this column or any of its children
+ val prefixMatchedUpdates = updates.filter(a => resolver(a.ref.head, col.name))
+ prefixMatchedUpdates match {
+ // if there is no exact match and no match for children, return the column as is
+ case updates if updates.isEmpty =>
+ col
+
+ // if there is an exact match, return the assigned expression
+ case Seq(update) if isExactMatch(update, col, resolver) =>
+ castIfNeeded(col, update.expr, resolver)
+
+ // if there are matches only for children
+ case updates if !hasExactMatch(updates, col, resolver) =>
+ col.dataType match {
+ case StructType(fields) =>
+ // build field expressions
+ val fieldExprs = fields.zipWithIndex.map { case (field, ordinal) =>
+ Alias(GetStructField(col, ordinal, Some(field.name)), field.name)()
+ }
+
+ // recursively apply this method on nested fields
+ val newUpdates = updates.map(u => u.copy(ref = u.ref.tail))
+ val updatedFieldExprs = applyUpdates(
+ ArraySeq.unsafeWrapArray(fieldExprs),
+ newUpdates,
+ resolver,
+ namePrefix :+ col.name)
+
+ // construct a new struct with updated field expressions
+ toNamedStruct(ArraySeq.unsafeWrapArray(fields), updatedFieldExprs)
+
+ case otherType =>
+ val colName = (namePrefix :+ col.name).mkString(".")
+ throw new AnalysisException(
+ "Updating nested fields is only supported for StructType " +
+ s"but $colName is of type $otherType"
+ )
+ }
+
+ // if there are conflicting updates, throw an exception
+ // there are two illegal scenarios:
+ // - multiple updates to the same column
+ // - updates to a top-level struct and its nested fields (e.g., a.b and a.b.c)
+ case updates if hasExactMatch(updates, col, resolver) =>
+ val conflictingCols = updates.map(u => (namePrefix ++ u.ref).mkString("."))
+ throw new AnalysisException(
+ "Updates are in conflict for these columns: " +
+ conflictingCols.distinct.mkString(", "))
+ }
+ }
+ }
+
+ private def toNamedStruct(fields: Seq[StructField], fieldExprs: Seq[Expression]): Expression = {
+ val namedStructExprs = fields.zip(fieldExprs).flatMap { case (field, expr) =>
+ Seq(Literal(field.name), expr)
+ }
+ CreateNamedStruct(namedStructExprs)
+ }
+
+ private def hasExactMatch(
+ updates: Seq[ColumnUpdate],
+ col: NamedExpression,
+ resolver: Resolver): Boolean = {
+
+ updates.exists(assignment => isExactMatch(assignment, col, resolver))
+ }
+
+ private def isExactMatch(
+ update: ColumnUpdate,
+ col: NamedExpression,
+ resolver: Resolver): Boolean = {
+
+ update.ref match {
+ case Seq(namePart) if resolver(namePart, col.name) => true
+ case _ => false
+ }
+ }
+
+ protected def castIfNeeded(
+ tableAttr: NamedExpression,
+ expr: Expression,
+ resolver: Resolver): Expression = {
+
+ val storeAssignmentPolicy = conf.storeAssignmentPolicy
+
+ // run the type check and catch type errors
+ storeAssignmentPolicy match {
+ case StoreAssignmentPolicy.STRICT | StoreAssignmentPolicy.ANSI =>
+ if (expr.nullable && !tableAttr.nullable) {
+ throw new AnalysisException(
+ s"Cannot write nullable values to non-null column '${tableAttr.name}'")
+ }
+
+ // use byName = true to catch cases when struct field names don't match
+ // e.g. a struct with fields (a, b) is assigned as a struct with fields (a, c) or (b, a)
+ val errors = new mutable.ArrayBuffer[String]()
+ val canWrite = DataType.canWrite(
+ expr.dataType, tableAttr.dataType, byName = true, resolver, tableAttr.name,
+ storeAssignmentPolicy, err => errors += err)
+
+ if (!canWrite) {
+ throw new AnalysisException(
+ s"Cannot write incompatible data:\n- ${errors.mkString("\n- ")}")
+ }
+
+ case _ => // OK
+ }
+
+ storeAssignmentPolicy match {
+ case _ if tableAttr.dataType.sameType(expr.dataType) =>
+ expr
+ case StoreAssignmentPolicy.ANSI =>
+ AnsiCast(expr, tableAttr.dataType, Option(conf.sessionLocalTimeZone))
+ case _ =>
+ Cast(expr, tableAttr.dataType, Option(conf.sessionLocalTimeZone))
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
new file mode 100644
index 000000000..70f6694af
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
+import org.apache.spark.sql.catalyst.plans.logical.InsertAction
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
+import org.apache.spark.sql.catalyst.rules.Rule
+
+/**
+ * A rule that checks MERGE operations contain only supported conditions.
+ *
+ * Note that this rule must be run in the resolution batch before Spark executes CheckAnalysis.
+ * Otherwise, CheckAnalysis will throw a less descriptive error.
+ */
+object CheckMergeIntoTableConditions extends Rule[LogicalPlan] {
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case m: MergeIntoIcebergTable if m.resolved =>
+ checkMergeIntoCondition("SEARCH", m.mergeCondition)
+
+ val actions = m.matchedActions ++ m.notMatchedActions
+ actions.foreach {
+ case DeleteAction(Some(cond)) => checkMergeIntoCondition("DELETE", cond)
+ case UpdateAction(Some(cond), _) => checkMergeIntoCondition("UPDATE", cond)
+ case InsertAction(Some(cond), _) => checkMergeIntoCondition("INSERT", cond)
+ case _ => // OK
+ }
+
+ m
+ }
+
+ private def checkMergeIntoCondition(condName: String, cond: Expression): Unit = {
+ if (!cond.deterministic) {
+ throw new AnalysisException(
+ s"Non-deterministic functions are not supported in $condName conditions of " +
+ s"MERGE operations: ${cond.sql}")
+ }
+
+ if (SubqueryExpression.hasSubquery(cond)) {
+ throw new AnalysisException(
+ s"Subqueries are not supported in conditions of MERGE operations. " +
+ s"Found a subquery in the $condName condition: ${cond.sql}")
+ }
+
+ if (cond.find(_.isInstanceOf[AggregateExpression]).isDefined) {
+ throw new AnalysisException(
+ s"Agg functions are not supported in $condName conditions of MERGE operations: " + {cond.sql})
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala
new file mode 100644
index 000000000..b3a9bda28
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable
+
+object MergeIntoIcebergTableResolutionCheck extends (LogicalPlan => Unit) {
+
+ override def apply(plan: LogicalPlan): Unit = {
+ plan foreach {
+ case m: UnresolvedMergeIntoIcebergTable =>
+ throw new AnalysisException(s"Could not resolve Iceberg MERGE INTO statement: $m")
+ case _ => // OK
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
new file mode 100644
index 000000000..7f0ca8fad
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.Cast
+import org.apache.spark.sql.catalyst.plans.logical.Call
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+
+object ProcedureArgumentCoercion extends Rule[LogicalPlan] {
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case c @ Call(procedure, args) if c.resolved =>
+ val params = procedure.parameters
+
+ val newArgs = args.zipWithIndex.map { case (arg, index) =>
+ val param = params(index)
+ val paramType = param.dataType
+ val argType = arg.dataType
+
+ if (paramType != argType && !Cast.canUpCast(argType, paramType)) {
+ throw new AnalysisException(
+ s"Wrong arg type for ${param.name}: cannot cast $argType to $paramType")
+ }
+
+ if (paramType != argType) {
+ Cast(arg, paramType)
+ } else {
+ arg
+ }
+ }
+
+ if (newArgs != args) {
+ c.copy(args = newArgs)
+ } else {
+ c
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
new file mode 100644
index 000000000..63ebdef95
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.plans.logical.Assignment
+import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
+import org.apache.spark.sql.catalyst.plans.logical.InsertAction
+import org.apache.spark.sql.catalyst.plans.logical.InsertStarAction
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
+import org.apache.spark.sql.catalyst.plans.logical.UpdateStarAction
+import org.apache.spark.sql.catalyst.rules.Rule
+
+/**
+ * A resolution rule similar to ResolveReferences in Spark but handles Iceberg MERGE operations.
+ */
+case class ResolveMergeIntoTableReferences(spark: SparkSession) extends Rule[LogicalPlan] {
+
+ private lazy val analyzer: Analyzer = spark.sessionState.analyzer
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
+ case m @ UnresolvedMergeIntoIcebergTable(targetTable, sourceTable, context)
+ if targetTable.resolved && sourceTable.resolved && m.duplicateResolved =>
+
+ val resolvedMatchedActions = context.matchedActions.map {
+ case DeleteAction(cond) =>
+ val resolvedCond = cond.map(resolveCond("DELETE", _, m))
+ DeleteAction(resolvedCond)
+
+ case UpdateAction(cond, assignments) =>
+ val resolvedCond = cond.map(resolveCond("UPDATE", _, m))
+ // the update action can access columns from both target and source tables
+ val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = false)
+ UpdateAction(resolvedCond, resolvedAssignments)
+
+ case UpdateStarAction(updateCondition) =>
+ val resolvedUpdateCondition = updateCondition.map(resolveCond("UPDATE", _, m))
+ val assignments = targetTable.output.map { attr =>
+ Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
+ }
+ // for UPDATE *, the value must be from the source table
+ val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
+ UpdateAction(resolvedUpdateCondition, resolvedAssignments)
+
+ case _ =>
+ throw new AnalysisException("Matched actions can only contain UPDATE or DELETE")
+ }
+
+ val resolvedNotMatchedActions = context.notMatchedActions.map {
+ case InsertAction(cond, assignments) =>
+ // the insert action is used when not matched, so its condition and value can only
+ // access columns from the source table
+ val resolvedCond = cond.map(resolveCond("INSERT", _, Project(Nil, m.sourceTable)))
+ val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
+ InsertAction(resolvedCond, resolvedAssignments)
+
+ case InsertStarAction(cond) =>
+ // the insert action is used when not matched, so its condition and value can only
+ // access columns from the source table
+ val resolvedCond = cond.map(resolveCond("INSERT", _, Project(Nil, m.sourceTable)))
+ val assignments = targetTable.output.map { attr =>
+ Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
+ }
+ val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
+ InsertAction(resolvedCond, resolvedAssignments)
+
+ case _ =>
+ throw new AnalysisException("Not matched actions can only contain INSERT")
+ }
+
+ val resolvedMergeCondition = resolveCond("SEARCH", context.mergeCondition, m)
+
+ MergeIntoIcebergTable(
+ targetTable,
+ sourceTable,
+ mergeCondition = resolvedMergeCondition,
+ matchedActions = resolvedMatchedActions,
+ notMatchedActions = resolvedNotMatchedActions)
+ }
+
+ private def resolveCond(condName: String, cond: Expression, plan: LogicalPlan): Expression = {
+ val resolvedCond = analyzer.resolveExpressionByPlanChildren(cond, plan)
+
+ val unresolvedAttrs = resolvedCond.references.filter(!_.resolved)
+ if (unresolvedAttrs.nonEmpty) {
+ throw new AnalysisException(
+ s"Cannot resolve ${unresolvedAttrs.map(_.sql).mkString("[", ",", "]")} in $condName condition " +
+ s"of MERGE operation given input columns: ${plan.inputSet.toSeq.map(_.sql).mkString("[", ",", "]")}")
+ }
+
+ resolvedCond
+ }
+
+ // copied from ResolveReferences in Spark
+ private def resolveAssignments(
+ assignments: Seq[Assignment],
+ mergeInto: UnresolvedMergeIntoIcebergTable,
+ resolveValuesWithSourceOnly: Boolean): Seq[Assignment] = {
+ assignments.map { assign =>
+ val resolvedKey = assign.key match {
+ case c if !c.resolved =>
+ resolveMergeExprOrFail(c, Project(Nil, mergeInto.targetTable))
+ case o => o
+ }
+ val resolvedValue = assign.value match {
+ // The update values may contain target and/or source references.
+ case c if !c.resolved =>
+ if (resolveValuesWithSourceOnly) {
+ resolveMergeExprOrFail(c, Project(Nil, mergeInto.sourceTable))
+ } else {
+ resolveMergeExprOrFail(c, mergeInto)
+ }
+ case o => o
+ }
+ Assignment(resolvedKey, resolvedValue)
+ }
+ }
+
+ // copied from ResolveReferences in Spark
+ private def resolveMergeExprOrFail(e: Expression, p: LogicalPlan): Expression = {
+ val resolved = analyzer.resolveExpressionByPlanChildren(e, p)
+ resolved.references.filter(!_.resolved).foreach { a =>
+ // Note: This will throw error only on unresolved attribute issues,
+ // not other resolution errors like mismatched data types.
+ val cols = p.inputSet.toSeq.map(_.sql).mkString(", ")
+ a.failAnalysis(s"cannot resolve ${a.sql} in MERGE command given columns [$cols]")
+ }
+ resolved
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
new file mode 100644
index 000000000..ee69b5e34
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import java.util.Locale
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.plans.logical.Call
+import org.apache.spark.sql.catalyst.plans.logical.CallArgument
+import org.apache.spark.sql.catalyst.plans.logical.CallStatement
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.NamedArgument
+import org.apache.spark.sql.catalyst.plans.logical.PositionalArgument
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.connector.catalog.CatalogPlugin
+import org.apache.spark.sql.connector.catalog.LookupCatalog
+import org.apache.spark.sql.connector.iceberg.catalog.ProcedureCatalog
+import org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter
+import scala.collection.Seq
+
+case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog {
+
+ protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case CallStatement(CatalogAndIdentifier(catalog, ident), args) =>
+ val procedure = catalog.asProcedureCatalog.loadProcedure(ident)
+
+ val params = procedure.parameters
+ val normalizedParams = normalizeParams(params)
+ validateParams(normalizedParams)
+
+ val normalizedArgs = normalizeArgs(args)
+ Call(procedure, args = buildArgExprs(normalizedParams, normalizedArgs).toSeq)
+ }
+
+ private def validateParams(params: Seq[ProcedureParameter]): Unit = {
+ // should not be any duplicate param names
+ val duplicateParamNames = params.groupBy(_.name).collect {
+ case (name, matchingParams) if matchingParams.length > 1 => name
+ }
+
+ if (duplicateParamNames.nonEmpty) {
+ throw new AnalysisException(s"Duplicate parameter names: ${duplicateParamNames.mkString("[", ",", "]")}")
+ }
+
+ // optional params should be at the end
+ params.sliding(2).foreach {
+ case Seq(previousParam, currentParam) if !previousParam.required && currentParam.required =>
+ throw new AnalysisException(
+ s"Optional parameters must be after required ones but $currentParam is after $previousParam")
+ case _ =>
+ }
+ }
+
+ private def buildArgExprs(
+ params: Seq[ProcedureParameter],
+ args: Seq[CallArgument]): Seq[Expression] = {
+
+ // build a map of declared parameter names to their positions
+ val nameToPositionMap = params.map(_.name).zipWithIndex.toMap
+
+ // build a map of parameter names to args
+ val nameToArgMap = buildNameToArgMap(params, args, nameToPositionMap)
+
+ // verify all required parameters are provided
+ val missingParamNames = params.filter(_.required).collect {
+ case param if !nameToArgMap.contains(param.name) => param.name
+ }
+
+ if (missingParamNames.nonEmpty) {
+ throw new AnalysisException(s"Missing required parameters: ${missingParamNames.mkString("[", ",", "]")}")
+ }
+
+ val argExprs = new Array[Expression](params.size)
+
+ nameToArgMap.foreach { case (name, arg) =>
+ val position = nameToPositionMap(name)
+ argExprs(position) = arg.expr
+ }
+
+ // assign nulls to optional params that were not set
+ params.foreach {
+ case p if !p.required && !nameToArgMap.contains(p.name) =>
+ val position = nameToPositionMap(p.name)
+ argExprs(position) = Literal.create(null, p.dataType)
+ case _ =>
+ }
+
+ argExprs
+ }
+
+ private def buildNameToArgMap(
+ params: Seq[ProcedureParameter],
+ args: Seq[CallArgument],
+ nameToPositionMap: Map[String, Int]): Map[String, CallArgument] = {
+
+ val containsNamedArg = args.exists(_.isInstanceOf[NamedArgument])
+ val containsPositionalArg = args.exists(_.isInstanceOf[PositionalArgument])
+
+ if (containsNamedArg && containsPositionalArg) {
+ throw new AnalysisException("Named and positional arguments cannot be mixed")
+ }
+
+ if (containsNamedArg) {
+ buildNameToArgMapUsingNames(args, nameToPositionMap)
+ } else {
+ buildNameToArgMapUsingPositions(args, params)
+ }
+ }
+
+ private def buildNameToArgMapUsingNames(
+ args: Seq[CallArgument],
+ nameToPositionMap: Map[String, Int]): Map[String, CallArgument] = {
+
+ val namedArgs = args.asInstanceOf[Seq[NamedArgument]]
+
+ val validationErrors = namedArgs.groupBy(_.name).collect {
+ case (name, matchingArgs) if matchingArgs.size > 1 => s"Duplicate procedure argument: $name"
+ case (name, _) if !nameToPositionMap.contains(name) => s"Unknown argument: $name"
+ }
+
+ if (validationErrors.nonEmpty) {
+ throw new AnalysisException(s"Could not build name to arg map: ${validationErrors.mkString(", ")}")
+ }
+
+ namedArgs.map(arg => arg.name -> arg).toMap
+ }
+
+ private def buildNameToArgMapUsingPositions(
+ args: Seq[CallArgument],
+ params: Seq[ProcedureParameter]): Map[String, CallArgument] = {
+
+ if (args.size > params.size) {
+ throw new AnalysisException("Too many arguments for procedure")
+ }
+
+ args.zipWithIndex.map { case (arg, position) =>
+ val param = params(position)
+ param.name -> arg
+ }.toMap
+ }
+
+ private def normalizeParams(params: Seq[ProcedureParameter]): Seq[ProcedureParameter] = {
+ params.map {
+ case param if param.required =>
+ val normalizedName = param.name.toLowerCase(Locale.ROOT)
+ ProcedureParameter.required(normalizedName, param.dataType)
+ case param =>
+ val normalizedName = param.name.toLowerCase(Locale.ROOT)
+ ProcedureParameter.optional(normalizedName, param.dataType)
+ }
+ }
+
+ private def normalizeArgs(args: Seq[CallArgument]): Seq[CallArgument] = {
+ args.map {
+ case a @ NamedArgument(name, _) => a.copy(name = name.toLowerCase(Locale.ROOT))
+ case other => other
+ }
+ }
+
+ implicit class CatalogHelper(plugin: CatalogPlugin) {
+ def asProcedureCatalog: ProcedureCatalog = plugin match {
+ case procedureCatalog: ProcedureCatalog =>
+ procedureCatalog
+ case _ =>
+ throw new AnalysisException(s"Cannot use catalog ${plugin.name}: not a ProcedureCatalog")
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
new file mode 100644
index 000000000..bbf7828eb
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.expressions.EqualNullSafe
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.Not
+import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
+import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
+import org.apache.spark.sql.connector.iceberg.catalog.SupportsRowLevelOperations
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation.Command.DELETE
+import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
+import org.apache.spark.sql.connector.write.RowLevelOperationTable
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+
+/**
+ * Assigns a rewrite plan for v2 tables that support rewriting data to handle DELETE statements.
+ *
+ * If a table implements SupportsDelete and SupportsRowLevelOperations, this rule assigns a rewrite
+ * plan but the optimizer will check whether this particular DELETE statement can be handled
+ * by simply passing delete filters to the connector. If yes, the optimizer will then discard
+ * the rewrite plan.
+ */
+object RewriteDeleteFromTable extends RewriteRowLevelCommand {
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case d @ DeleteFromIcebergTable(aliasedTable, Some(cond), None) if d.resolved =>
+ EliminateSubqueryAliases(aliasedTable) match {
+ case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) =>
+ val operation = buildRowLevelOperation(tbl, DELETE)
+ val table = RowLevelOperationTable(tbl, operation)
+ val rewritePlan = operation match {
+ case _: SupportsDelta =>
+ buildWriteDeltaPlan(r, table, cond)
+ case _ =>
+ buildReplaceDataPlan(r, table, cond)
+ }
+ // keep the original relation in DELETE to try deleting using filters
+ DeleteFromIcebergTable(r, Some(cond), Some(rewritePlan))
+
+ case p =>
+ throw new AnalysisException(s"$p is not an Iceberg table")
+ }
+ }
+
+ // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
+ private def buildReplaceDataPlan(
+ relation: DataSourceV2Relation,
+ operationTable: RowLevelOperationTable,
+ cond: Expression): ReplaceData = {
+
+ // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
+ val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
+
+ // construct a read relation and include all required metadata columns
+ val readRelation = buildReadRelation(relation, operationTable, metadataAttrs)
+
+ // construct a plan that contains unmatched rows in matched groups that must be carried over
+ // such rows do not match the condition but have to be copied over as the source can replace
+ // only groups of rows
+ val remainingRowsFilter = Not(EqualNullSafe(cond, Literal.TrueLiteral))
+ val remainingRowsPlan = Filter(remainingRowsFilter, readRelation)
+
+ // build a plan to replace read groups in the table
+ val writeRelation = relation.copy(table = operationTable)
+ ReplaceData(writeRelation, remainingRowsPlan, relation)
+ }
+
+ // build a rewrite plan for sources that support row deltas
+ private def buildWriteDeltaPlan(
+ relation: DataSourceV2Relation,
+ operationTable: RowLevelOperationTable,
+ cond: Expression): WriteDelta = {
+
+ // resolve all needed attrs (e.g. row ID and any required metadata attrs)
+ val rowIdAttrs = resolveRowIdAttrs(relation, operationTable.operation)
+ val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
+
+ // construct a read relation and include all required metadata columns
+ val readRelation = buildReadRelation(relation, operationTable, metadataAttrs, rowIdAttrs)
+
+ // construct a plan that only contains records to delete
+ val deletedRowsPlan = Filter(cond, readRelation)
+ val operationType = Alias(Literal(DELETE_OPERATION), OPERATION_COLUMN)()
+ val requiredWriteAttrs = dedupAttrs(rowIdAttrs ++ metadataAttrs)
+ val project = Project(operationType +: requiredWriteAttrs, deletedRowsPlan)
+
+ // build a plan to write deletes to the table
+ val writeRelation = relation.copy(table = operationTable)
+ val projections = buildWriteDeltaProjections(project, Nil, rowIdAttrs, metadataAttrs)
+ WriteDelta(writeRelation, project, relation, projections)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
new file mode 100644
index 000000000..886ed5002
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
+import org.apache.spark.sql.catalyst.expressions.IsNotNull
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
+import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
+import org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID
+import org.apache.spark.sql.catalyst.plans.FullOuter
+import org.apache.spark.sql.catalyst.plans.Inner
+import org.apache.spark.sql.catalyst.plans.LeftAnti
+import org.apache.spark.sql.catalyst.plans.LeftOuter
+import org.apache.spark.sql.catalyst.plans.RightOuter
+import org.apache.spark.sql.catalyst.plans.logical.AppendData
+import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
+import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.plans.logical.HintInfo
+import org.apache.spark.sql.catalyst.plans.logical.InsertAction
+import org.apache.spark.sql.catalyst.plans.logical.Join
+import org.apache.spark.sql.catalyst.plans.logical.JoinHint
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeAction
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.MergeRows
+import org.apache.spark.sql.catalyst.plans.logical.NO_BROADCAST_HASH
+import org.apache.spark.sql.catalyst.plans.logical.NoStatsUnaryNode
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
+import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
+import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.NamedReference
+import org.apache.spark.sql.connector.iceberg.catalog.SupportsRowLevelOperations
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation.Command.MERGE
+import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
+import org.apache.spark.sql.connector.write.RowLevelOperationTable
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.types.IntegerType
+
+/**
+ * Assigns a rewrite plan for v2 tables that support rewriting data to handle MERGE statements.
+ *
+ * This rule assumes the commands have been fully resolved and all assignments have been aligned.
+ * That's why it must be run after AlignRowLevelCommandAssignments.
+ */
+object RewriteMergeIntoTable extends RewriteRowLevelCommand {
+
+ private final val ROW_FROM_SOURCE = "__row_from_source"
+ private final val ROW_FROM_TARGET = "__row_from_target"
+ private final val ROW_ID = "__row_id"
+
+ private final val ROW_FROM_SOURCE_REF = FieldReference(ROW_FROM_SOURCE)
+ private final val ROW_FROM_TARGET_REF = FieldReference(ROW_FROM_TARGET)
+ private final val ROW_ID_REF = FieldReference(ROW_ID)
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
+ if m.resolved && m.aligned && matchedActions.isEmpty && notMatchedActions.size == 1 =>
+
+ EliminateSubqueryAliases(aliasedTable) match {
+ case r: DataSourceV2Relation =>
+ // NOT MATCHED conditions may only refer to columns in source so they can be pushed down
+ val insertAction = notMatchedActions.head.asInstanceOf[InsertAction]
+ val filteredSource = insertAction.condition match {
+ case Some(insertCond) => Filter(insertCond, source)
+ case None => source
+ }
+
+ // when there are no MATCHED actions, use a left anti join to remove any matching rows
+ // and switch to using a regular append instead of a row-level merge
+ // only unmatched source rows that match the condition are appended to the table
+ val joinPlan = Join(filteredSource, r, LeftAnti, Some(cond), JoinHint.NONE)
+
+ val outputExprs = insertAction.assignments.map(_.value)
+ val outputColNames = r.output.map(_.name)
+ val outputCols = outputExprs.zip(outputColNames).map { case (expr, name) =>
+ Alias(expr, name)()
+ }
+ val project = Project(outputCols, joinPlan)
+
+ AppendData.byPosition(r, project)
+
+ case p =>
+ throw new AnalysisException(s"$p is not an Iceberg table")
+ }
+
+ case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
+ if m.resolved && m.aligned && matchedActions.isEmpty =>
+
+ EliminateSubqueryAliases(aliasedTable) match {
+ case r: DataSourceV2Relation =>
+ // when there are no MATCHED actions, use a left anti join to remove any matching rows
+ // and switch to using a regular append instead of a row-level merge
+ // only unmatched source rows that match action conditions are appended to the table
+ val joinPlan = Join(source, r, LeftAnti, Some(cond), JoinHint.NONE)
+
+ val notMatchedConditions = notMatchedActions.map(actionCondition)
+ val notMatchedOutputs = notMatchedActions.map(actionOutput(_, Nil))
+
+ // merge rows as there are multiple not matched actions
+ val mergeRows = MergeRows(
+ isSourceRowPresent = TrueLiteral,
+ isTargetRowPresent = FalseLiteral,
+ matchedConditions = Nil,
+ matchedOutputs = Nil,
+ notMatchedConditions = notMatchedConditions,
+ notMatchedOutputs = notMatchedOutputs,
+ targetOutput = Nil,
+ rowIdAttrs = Nil,
+ performCardinalityCheck = false,
+ emitNotMatchedTargetRows = false,
+ output = buildMergeRowsOutput(Nil, notMatchedOutputs, r.output),
+ joinPlan)
+
+ AppendData.byPosition(r, mergeRows)
+
+ case p =>
+ throw new AnalysisException(s"$p is not an Iceberg table")
+ }
+
+ case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
+ if m.resolved && m.aligned =>
+
+ EliminateSubqueryAliases(aliasedTable) match {
+ case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) =>
+ val operation = buildRowLevelOperation(tbl, MERGE)
+ val table = RowLevelOperationTable(tbl, operation)
+ val rewritePlan = operation match {
+ case _: SupportsDelta =>
+ buildWriteDeltaPlan(r, table, source, cond, matchedActions, notMatchedActions)
+ case _ =>
+ buildReplaceDataPlan(r, table, source, cond, matchedActions, notMatchedActions)
+ }
+
+ m.copy(rewritePlan = Some(rewritePlan))
+
+ case p =>
+ throw new AnalysisException(s"$p is not an Iceberg table")
+ }
+ }
+
+ // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
+ private def buildReplaceDataPlan(
+ relation: DataSourceV2Relation,
+ operationTable: RowLevelOperationTable,
+ source: LogicalPlan,
+ cond: Expression,
+ matchedActions: Seq[MergeAction],
+ notMatchedActions: Seq[MergeAction]): ReplaceData = {
+
+ // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
+ val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
+
+ // construct a scan relation and include all required metadata columns
+ val readRelation = buildReadRelation(relation, operationTable, metadataAttrs)
+ val readAttrs = readRelation.output
+
+ // project an extra column to check if a target row exists after the join
+ // project a synthetic row ID to perform the cardinality check
+ val rowFromTarget = Alias(TrueLiteral, ROW_FROM_TARGET)()
+ val rowId = Alias(MonotonicallyIncreasingID(), ROW_ID)()
+ val targetTableProjExprs = readAttrs ++ Seq(rowFromTarget, rowId)
+ val targetTableProj = Project(targetTableProjExprs, readRelation)
+
+ // project an extra column to check if a source row exists after the join
+ val rowFromSource = Alias(TrueLiteral, ROW_FROM_SOURCE)()
+ val sourceTableProjExprs = source.output :+ rowFromSource
+ val sourceTableProj = Project(sourceTableProjExprs, source)
+
+ // use left outer join if there is no NOT MATCHED action, unmatched source rows can be discarded
+ // use full outer join in all other cases, unmatched source rows may be needed
+ // disable broadcasts for the target table to perform the cardinality check
+ val joinType = if (notMatchedActions.isEmpty) LeftOuter else FullOuter
+ val joinHint = JoinHint(leftHint = Some(HintInfo(Some(NO_BROADCAST_HASH))), rightHint = None)
+ val joinPlan = Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(cond), joinHint)
+
+ // add an extra matched action to output the original row if none of the actual actions matched
+ // this is needed to keep target rows that should be copied over
+ val matchedConditions = matchedActions.map(actionCondition) :+ TrueLiteral
+ val matchedOutputs = matchedActions.map(actionOutput(_, metadataAttrs)) :+ readAttrs
+
+ val notMatchedConditions = notMatchedActions.map(actionCondition)
+ val notMatchedOutputs = notMatchedActions.map(actionOutput(_, metadataAttrs))
+
+ val rowIdAttr = resolveAttrRef(ROW_ID_REF, joinPlan)
+ val rowFromSourceAttr = resolveAttrRef(ROW_FROM_SOURCE_REF, joinPlan)
+ val rowFromTargetAttr = resolveAttrRef(ROW_FROM_TARGET_REF, joinPlan)
+
+ val mergeRows = MergeRows(
+ isSourceRowPresent = IsNotNull(rowFromSourceAttr),
+ isTargetRowPresent = if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
+ matchedConditions = matchedConditions,
+ matchedOutputs = matchedOutputs,
+ notMatchedConditions = notMatchedConditions,
+ notMatchedOutputs = notMatchedOutputs,
+ targetOutput = readAttrs,
+ rowIdAttrs = Seq(rowIdAttr),
+ performCardinalityCheck = isCardinalityCheckNeeded(matchedActions),
+ emitNotMatchedTargetRows = true,
+ output = buildMergeRowsOutput(matchedOutputs, notMatchedOutputs, readAttrs),
+ joinPlan)
+
+ // build a plan to replace read groups in the table
+ val writeRelation = relation.copy(table = operationTable)
+ ReplaceData(writeRelation, mergeRows, relation)
+ }
+
+ // build a rewrite plan for sources that support row deltas
+ private def buildWriteDeltaPlan(
+ relation: DataSourceV2Relation,
+ operationTable: RowLevelOperationTable,
+ source: LogicalPlan,
+ cond: Expression,
+ matchedActions: Seq[MergeAction],
+ notMatchedActions: Seq[MergeAction]): WriteDelta = {
+
+ // resolve all needed attrs (e.g. row ID and any required metadata attrs)
+ val rowAttrs = relation.output
+ val rowIdAttrs = resolveRowIdAttrs(relation, operationTable.operation)
+ val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
+
+ // construct a scan relation and include all required metadata columns
+ val readRelation = buildReadRelation(relation, operationTable, metadataAttrs, rowIdAttrs)
+ val readAttrs = readRelation.output
+
+ // project an extra column to check if a target row exists after the join
+ val targetTableProjExprs = readAttrs :+ Alias(TrueLiteral, ROW_FROM_TARGET)()
+ val targetTableProj = Project(targetTableProjExprs, readRelation)
+
+ // project an extra column to check if a source row exists after the join
+ val sourceTableProjExprs = source.output :+ Alias(TrueLiteral, ROW_FROM_SOURCE)()
+ val sourceTableProj = Project(sourceTableProjExprs, source)
+
+ // use inner join if there is no NOT MATCHED action, unmatched source rows can be discarded
+ // use right outer join in all other cases, unmatched source rows may be needed
+ // also disable broadcasts for the target table to perform the cardinality check
+ val joinType = if (notMatchedActions.isEmpty) Inner else RightOuter
+ val joinHint = JoinHint(leftHint = Some(HintInfo(Some(NO_BROADCAST_HASH))), rightHint = None)
+ val joinPlan = Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(cond), joinHint)
+
+ val deleteRowValues = buildDeltaDeleteRowValues(rowAttrs, rowIdAttrs)
+ val metadataReadAttrs = readAttrs.filterNot(relation.outputSet.contains)
+
+ val matchedConditions = matchedActions.map(actionCondition)
+ val matchedOutputs = matchedActions.map(deltaActionOutput(_, deleteRowValues, metadataReadAttrs))
+
+ val notMatchedConditions = notMatchedActions.map(actionCondition)
+ val notMatchedOutputs = notMatchedActions.map(deltaActionOutput(_, deleteRowValues, metadataReadAttrs))
+
+ val operationTypeAttr = AttributeReference(OPERATION_COLUMN, IntegerType, nullable = false)()
+ val rowFromSourceAttr = resolveAttrRef(ROW_FROM_SOURCE_REF, joinPlan)
+ val rowFromTargetAttr = resolveAttrRef(ROW_FROM_TARGET_REF, joinPlan)
+
+ // merged rows must contain values for the operation type and all read attrs
+ val mergeRowsOutput = buildMergeRowsOutput(matchedOutputs, notMatchedOutputs, operationTypeAttr +: readAttrs)
+
+ val mergeRows = MergeRows(
+ isSourceRowPresent = IsNotNull(rowFromSourceAttr),
+ isTargetRowPresent = if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
+ matchedConditions = matchedConditions,
+ matchedOutputs = matchedOutputs,
+ notMatchedConditions = notMatchedConditions,
+ notMatchedOutputs = notMatchedOutputs,
+ // only needed if emitting unmatched target rows
+ targetOutput = Nil,
+ rowIdAttrs = rowIdAttrs,
+ performCardinalityCheck = isCardinalityCheckNeeded(matchedActions),
+ emitNotMatchedTargetRows = false,
+ output = mergeRowsOutput,
+ joinPlan)
+
+ // build a plan to write the row delta to the table
+ val writeRelation = relation.copy(table = operationTable)
+ val projections = buildWriteDeltaProjections(mergeRows, rowAttrs, rowIdAttrs, metadataAttrs)
+ WriteDelta(writeRelation, mergeRows, relation, projections)
+ }
+
+ private def actionCondition(action: MergeAction): Expression = {
+ action.condition.getOrElse(TrueLiteral)
+ }
+
+ private def actionOutput(
+ clause: MergeAction,
+ metadataAttrs: Seq[Attribute]): Seq[Expression] = {
+
+ clause match {
+ case u: UpdateAction =>
+ u.assignments.map(_.value) ++ metadataAttrs
+
+ case _: DeleteAction =>
+ Nil
+
+ case i: InsertAction =>
+ i.assignments.map(_.value) ++ metadataAttrs.map(attr => Literal(null, attr.dataType))
+
+ case other =>
+ throw new AnalysisException(s"Unexpected action: $other")
+ }
+ }
+
+ private def deltaActionOutput(
+ action: MergeAction,
+ deleteRowValues: Seq[Expression],
+ metadataAttrs: Seq[Attribute]): Seq[Expression] = {
+
+ action match {
+ case u: UpdateAction =>
+ Seq(Literal(UPDATE_OPERATION)) ++ u.assignments.map(_.value) ++ metadataAttrs
+
+ case _: DeleteAction =>
+ Seq(Literal(DELETE_OPERATION)) ++ deleteRowValues ++ metadataAttrs
+
+ case i: InsertAction =>
+ val metadataAttrValues = metadataAttrs.map(attr => Literal(null, attr.dataType))
+ Seq(Literal(INSERT_OPERATION)) ++ i.assignments.map(_.value) ++ metadataAttrValues
+
+ case other =>
+ throw new AnalysisException(s"Unexpected action: $other")
+ }
+ }
+
+ private def buildMergeRowsOutput(
+ matchedOutputs: Seq[Seq[Expression]],
+ notMatchedOutputs: Seq[Seq[Expression]],
+ attrs: Seq[Attribute]): Seq[Attribute] = {
+
+ // collect all outputs from matched and not matched actions (ignoring DELETEs)
+ val outputs = matchedOutputs.filter(_.nonEmpty) ++ notMatchedOutputs.filter(_.nonEmpty)
+
+ // build a correct nullability map for output attributes
+ // an attribute is nullable if at least one matched or not matched action may produce null
+ val nullabilityMap = attrs.indices.map { index =>
+ index -> outputs.exists(output => output(index).nullable)
+ }.toMap
+
+ attrs.zipWithIndex.map { case (attr, index) =>
+ attr.withNullability(nullabilityMap(index))
+ }
+ }
+
+ private def isCardinalityCheckNeeded(actions: Seq[MergeAction]): Boolean = actions match {
+ case Seq(DeleteAction(None)) => false
+ case _ => true
+ }
+
+ private def buildDeltaDeleteRowValues(
+ rowAttrs: Seq[Attribute],
+ rowIdAttrs: Seq[Attribute]): Seq[Expression] = {
+
+ // nullify all row attrs that are not part of the row ID
+ val rowIdAttSet = AttributeSet(rowIdAttrs)
+ rowAttrs.map {
+ case attr if rowIdAttSet.contains(attr) => attr
+ case attr => Literal(null, attr.dataType)
+ }
+ }
+
+ private def resolveAttrRef(ref: NamedReference, plan: LogicalPlan): AttributeReference = {
+ ExtendedV2ExpressionUtils.resolveRef[AttributeReference](ref, plan)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelCommand.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelCommand.scala
new file mode 100644
index 000000000..e9af76c22
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelCommand.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.ProjectingInternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.ExprId
+import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.util.WriteDeltaProjections
+import org.apache.spark.sql.connector.iceberg.catalog.SupportsRowLevelOperations
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation.Command
+import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
+import org.apache.spark.sql.connector.write.RowLevelOperationInfoImpl
+import org.apache.spark.sql.connector.write.RowLevelOperationTable
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import scala.collection.compat.immutable.ArraySeq
+import scala.collection.mutable
+
+trait RewriteRowLevelCommand extends Rule[LogicalPlan] {
+
+ protected def buildRowLevelOperation(
+ table: SupportsRowLevelOperations,
+ command: Command): RowLevelOperation = {
+ val info = RowLevelOperationInfoImpl(command, CaseInsensitiveStringMap.empty())
+ val builder = table.newRowLevelOperationBuilder(info)
+ builder.build()
+ }
+
+ protected def buildReadRelation(
+ relation: DataSourceV2Relation,
+ table: RowLevelOperationTable,
+ metadataAttrs: Seq[AttributeReference],
+ rowIdAttrs: Seq[AttributeReference] = Nil): DataSourceV2Relation = {
+
+ val attrs = dedupAttrs(relation.output ++ rowIdAttrs ++ metadataAttrs)
+ relation.copy(table = table, output = attrs)
+ }
+
+ protected def dedupAttrs(attrs: Seq[AttributeReference]): Seq[AttributeReference] = {
+ val exprIds = mutable.Set.empty[ExprId]
+ attrs.flatMap { attr =>
+ if (exprIds.contains(attr.exprId)) {
+ None
+ } else {
+ exprIds += attr.exprId
+ Some(attr)
+ }
+ }
+ }
+
+ protected def buildWriteDeltaProjections(
+ plan: LogicalPlan,
+ rowAttrs: Seq[Attribute],
+ rowIdAttrs: Seq[Attribute],
+ metadataAttrs: Seq[Attribute]): WriteDeltaProjections = {
+
+ val rowProjection = if (rowAttrs.nonEmpty) {
+ Some(newLazyProjection(plan, rowAttrs, usePlanTypes = true))
+ } else {
+ None
+ }
+
+ // in MERGE, the plan may contain both delete and insert records that may affect
+ // the nullability of metadata columns (e.g. metadata columns for new records are always null)
+ // since metadata columns are never passed with new records to insert,
+ // use the actual metadata column types instead of the ones present in the plan
+
+ val rowIdProjection = newLazyProjection(plan, rowIdAttrs, usePlanTypes = false)
+
+ val metadataProjection = if (metadataAttrs.nonEmpty) {
+ Some(newLazyProjection(plan, metadataAttrs, usePlanTypes = false))
+ } else {
+ None
+ }
+
+ WriteDeltaProjections(rowProjection, rowIdProjection, metadataProjection)
+ }
+
+ // the projection is done by name, ignoring expr IDs
+ private def newLazyProjection(
+ plan: LogicalPlan,
+ attrs: Seq[Attribute],
+ usePlanTypes: Boolean): ProjectingInternalRow = {
+
+ val colOrdinals = attrs.map(attr => plan.output.indexWhere(_.name == attr.name))
+ val schema = if (usePlanTypes) {
+ val planAttrs = colOrdinals.map(plan.output(_))
+ StructType.fromAttributes(planAttrs)
+ } else {
+ StructType.fromAttributes(attrs)
+ }
+ ProjectingInternalRow(schema, colOrdinals)
+ }
+
+ protected def resolveRequiredMetadataAttrs(
+ relation: DataSourceV2Relation,
+ operation: RowLevelOperation): Seq[AttributeReference] = {
+
+ ExtendedV2ExpressionUtils.resolveRefs[AttributeReference](
+ operation.requiredMetadataAttributes.toSeq,
+ relation)
+ }
+
+ protected def resolveRowIdAttrs(
+ relation: DataSourceV2Relation,
+ operation: RowLevelOperation): Seq[AttributeReference] = {
+
+ operation match {
+ case supportsDelta: SupportsDelta =>
+ val rowIdAttrs = ExtendedV2ExpressionUtils.resolveRefs[AttributeReference](
+ supportsDelta.rowId.toSeq,
+ relation)
+
+ val nullableRowIdAttrs = rowIdAttrs.filter(_.nullable)
+ if (nullableRowIdAttrs.nonEmpty) {
+ throw new AnalysisException(s"Row ID attrs cannot be nullable: $nullableRowIdAttrs")
+ }
+
+ rowIdAttrs
+
+ case other =>
+ throw new AnalysisException(s"Operation $other does not support deltas")
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala
new file mode 100644
index 000000000..110b38d31
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.expressions.EqualNullSafe
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.If
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.Not
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.plans.logical.Assignment
+import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.plans.logical.Union
+import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
+import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
+import org.apache.spark.sql.connector.iceberg.catalog.SupportsRowLevelOperations
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation.Command.UPDATE
+import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
+import org.apache.spark.sql.connector.write.RowLevelOperationTable
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+
+/**
+ * Assigns a rewrite plan for v2 tables that support rewriting data to handle UPDATE statements.
+ *
+ * This rule assumes the commands have been fully resolved and all assignments have been aligned.
+ * That's why it must be run after AlignRowLevelCommandAssignments.
+ *
+ * This rule also must be run in the same batch with DeduplicateRelations in Spark.
+ */
+object RewriteUpdateTable extends RewriteRowLevelCommand {
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case u @ UpdateIcebergTable(aliasedTable, assignments, cond, None) if u.resolved && u.aligned =>
+ EliminateSubqueryAliases(aliasedTable) match {
+ case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) =>
+ val operation = buildRowLevelOperation(tbl, UPDATE)
+ val table = RowLevelOperationTable(tbl, operation)
+ val updateCond = cond.getOrElse(Literal.TrueLiteral)
+ val rewritePlan = operation match {
+ case _: SupportsDelta =>
+ buildWriteDeltaPlan(r, table, assignments, updateCond)
+ case _ if SubqueryExpression.hasSubquery(updateCond) =>
+ buildReplaceDataWithUnionPlan(r, table, assignments, updateCond)
+ case _ =>
+ buildReplaceDataPlan(r, table, assignments, updateCond)
+ }
+ UpdateIcebergTable(r, assignments, cond, Some(rewritePlan))
+
+ case p =>
+ throw new AnalysisException(s"$p is not an Iceberg table")
+ }
+ }
+
+ // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
+ // if the condition does NOT contain a subquery
+ private def buildReplaceDataPlan(
+ relation: DataSourceV2Relation,
+ operationTable: RowLevelOperationTable,
+ assignments: Seq[Assignment],
+ cond: Expression): ReplaceData = {
+
+ // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
+ val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
+
+ // construct a read relation and include all required metadata columns
+ val readRelation = buildReadRelation(relation, operationTable, metadataAttrs)
+
+ // build a plan with updated and copied over records
+ val updatedAndRemainingRowsPlan = buildUpdateProjection(readRelation, assignments, cond)
+
+ // build a plan to replace read groups in the table
+ val writeRelation = relation.copy(table = operationTable)
+ ReplaceData(writeRelation, updatedAndRemainingRowsPlan, relation)
+ }
+
+ // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
+ // if the condition contains a subquery
+ private def buildReplaceDataWithUnionPlan(
+ relation: DataSourceV2Relation,
+ operationTable: RowLevelOperationTable,
+ assignments: Seq[Assignment],
+ cond: Expression): ReplaceData = {
+
+ // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
+ val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
+
+ // construct a read relation and include all required metadata columns
+ // the same read relation will be used to read records that must be updated and be copied over
+ // DeduplicateRelations will take care of duplicated attr IDs
+ val readRelation = buildReadRelation(relation, operationTable, metadataAttrs)
+
+ // build a plan for records that match the cond and should be updated
+ val matchedRowsPlan = Filter(cond, readRelation)
+ val updatedRowsPlan = buildUpdateProjection(matchedRowsPlan, assignments)
+
+ // build a plan for records that did not match the cond but had to be copied over
+ val remainingRowFilter = Not(EqualNullSafe(cond, Literal.TrueLiteral))
+ val remainingRowsPlan = Filter(remainingRowFilter, readRelation)
+
+ // new state is a union of updated and copied over records
+ val updatedAndRemainingRowsPlan = Union(updatedRowsPlan, remainingRowsPlan)
+
+ // build a plan to replace read groups in the table
+ val writeRelation = relation.copy(table = operationTable)
+ ReplaceData(writeRelation, updatedAndRemainingRowsPlan, relation)
+ }
+
+ // build a rewrite plan for sources that support row deltas
+ private def buildWriteDeltaPlan(
+ relation: DataSourceV2Relation,
+ operationTable: RowLevelOperationTable,
+ assignments: Seq[Assignment],
+ cond: Expression): WriteDelta = {
+
+ // resolve all needed attrs (e.g. row ID and any required metadata attrs)
+ val rowAttrs = relation.output
+ val rowIdAttrs = resolveRowIdAttrs(relation, operationTable.operation)
+ val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
+
+ // construct a scan relation and include all required metadata columns
+ val readRelation = buildReadRelation(relation, operationTable, metadataAttrs, rowIdAttrs)
+
+ // build a plan for updated records that match the cond
+ val matchedRowsPlan = Filter(cond, readRelation)
+ val updatedRowsPlan = buildUpdateProjection(matchedRowsPlan, assignments)
+ val operationType = Alias(Literal(UPDATE_OPERATION), OPERATION_COLUMN)()
+ val project = Project(operationType +: updatedRowsPlan.output, updatedRowsPlan)
+
+ // build a plan to write the row delta to the table
+ val writeRelation = relation.copy(table = operationTable)
+ val projections = buildWriteDeltaProjections(project, rowAttrs, rowIdAttrs, metadataAttrs)
+ WriteDelta(writeRelation, project, relation, projections)
+ }
+
+ // this method assumes the assignments have been already aligned before
+ // the condition passed to this method may be different from the UPDATE condition
+ private def buildUpdateProjection(
+ plan: LogicalPlan,
+ assignments: Seq[Assignment],
+ cond: Expression = Literal.TrueLiteral): LogicalPlan = {
+
+ // TODO: avoid executing the condition for each column
+
+ // the plan output may include metadata columns that are not modified
+ // that's why the number of assignments may not match the number of plan output columns
+
+ val assignedValues = assignments.map(_.value)
+ val updatedValues = plan.output.zipWithIndex.map { case (attr, index) =>
+ if (index < assignments.size) {
+ val assignedExpr = assignedValues(index)
+ val updatedValue = If(cond, assignedExpr, attr)
+ Alias(updatedValue, attr.name)()
+ } else {
+ attr
+ }
+ }
+
+ Project(updatedValues, plan)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala
new file mode 100644
index 000000000..ce3818922
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.SQLConfHelper
+import org.apache.spark.sql.catalyst.plans.logical.Assignment
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.types.DataType
+
+object AssignmentUtils extends SQLConfHelper {
+
+ /**
+ * Checks whether assignments are aligned and match table columns.
+ *
+ * @param table a target table
+ * @param assignments assignments to check
+ * @return true if the assignments are aligned
+ */
+ def aligned(table: LogicalPlan, assignments: Seq[Assignment]): Boolean = {
+ val sameSize = table.output.size == assignments.size
+ sameSize && table.output.zip(assignments).forall { case (attr, assignment) =>
+ val key = assignment.key
+ val value = assignment.value
+ val refsEqual = toAssignmentRef(attr).zip(toAssignmentRef(key))
+ .forall{ case (attrRef, keyRef) => conf.resolver(attrRef, keyRef)}
+
+ refsEqual &&
+ DataType.equalsIgnoreCompatibleNullability(value.dataType, attr.dataType) &&
+ (attr.nullable || !value.nullable)
+ }
+ }
+
+ def toAssignmentRef(expr: Expression): Seq[String] = expr match {
+ case attr: AttributeReference =>
+ Seq(attr.name)
+ case Alias(child, _) =>
+ toAssignmentRef(child)
+ case GetStructField(child, _, Some(name)) =>
+ toAssignmentRef(child) :+ name
+ case other: ExtractValue =>
+ throw new AnalysisException(s"Updating nested fields is only supported for structs: $other")
+ case other =>
+ throw new AnalysisException(s"Cannot convert to a reference, unsupported expression: $other")
+ }
+
+ def handleCharVarcharLimits(assignment: Assignment): Assignment = {
+ val key = assignment.key
+ val value = assignment.value
+
+ val rawKeyType = key.transform {
+ case attr: AttributeReference =>
+ CharVarcharUtils.getRawType(attr.metadata)
+ .map(attr.withDataType)
+ .getOrElse(attr)
+ }.dataType
+
+ if (CharVarcharUtils.hasCharVarchar(rawKeyType)) {
+ val newKey = key.transform {
+ case attr: AttributeReference => CharVarcharUtils.cleanAttrMetadata(attr)
+ }
+ val newValue = CharVarcharUtils.stringLengthCheck(value, rawKeyType)
+ Assignment(newKey, newValue)
+ } else {
+ assignment
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtendedV2ExpressionUtils.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtendedV2ExpressionUtils.scala
new file mode 100644
index 000000000..b872281d7
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtendedV2ExpressionUtils.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.SQLConfHelper
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.connector.expressions.{Expression => V2Expression}
+import org.apache.spark.sql.connector.expressions.{SortDirection => V2SortDirection}
+import org.apache.spark.sql.connector.expressions.{NullOrdering => V2NullOrdering}
+import org.apache.spark.sql.connector.expressions.BucketTransform
+import org.apache.spark.sql.connector.expressions.DaysTransform
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.HoursTransform
+import org.apache.spark.sql.connector.expressions.IdentityTransform
+import org.apache.spark.sql.connector.expressions.MonthsTransform
+import org.apache.spark.sql.connector.expressions.NamedReference
+import org.apache.spark.sql.connector.expressions.SortValue
+import org.apache.spark.sql.connector.expressions.TruncateTransform
+import org.apache.spark.sql.connector.expressions.YearsTransform
+import org.apache.spark.sql.errors.QueryCompilationErrors
+
+/**
+ * A class that is inspired by V2ExpressionUtils in Spark but supports Iceberg transforms.
+ */
+object ExtendedV2ExpressionUtils extends SQLConfHelper {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+
+ def resolveRef[T <: NamedExpression](ref: NamedReference, plan: LogicalPlan): T = {
+ plan.resolve(ref.fieldNames.toSeq, conf.resolver) match {
+ case Some(namedExpr) =>
+ namedExpr.asInstanceOf[T]
+ case None =>
+ val name = ref.fieldNames.toSeq.quoted
+ val outputString = plan.output.map(_.name).mkString(",")
+ throw QueryCompilationErrors.cannotResolveAttributeError(name, outputString)
+ }
+ }
+
+ def resolveRefs[T <: NamedExpression](refs: Seq[NamedReference], plan: LogicalPlan): Seq[T] = {
+ refs.map(ref => resolveRef[T](ref, plan))
+ }
+
+ def toCatalyst(expr: V2Expression, query: LogicalPlan): Expression = {
+ expr match {
+ case SortValue(child, direction, nullOrdering) =>
+ val catalystChild = toCatalyst(child, query)
+ SortOrder(catalystChild, toCatalyst(direction), toCatalyst(nullOrdering), Seq.empty)
+ case IdentityTransform(ref) =>
+ resolveRef[NamedExpression](ref, query)
+ case BucketTransform(numBuckets, ref) =>
+ IcebergBucketTransform(numBuckets, resolveRef[NamedExpression](ref, query))
+ case TruncateTransform(length, ref) =>
+ IcebergTruncateTransform(resolveRef[NamedExpression](ref, query), length)
+ case YearsTransform(ref) =>
+ IcebergYearTransform(resolveRef[NamedExpression](ref, query))
+ case MonthsTransform(ref) =>
+ IcebergMonthTransform(resolveRef[NamedExpression](ref, query))
+ case DaysTransform(ref) =>
+ IcebergDayTransform(resolveRef[NamedExpression](ref, query))
+ case HoursTransform(ref) =>
+ IcebergHourTransform(resolveRef[NamedExpression](ref, query))
+ case ref: FieldReference =>
+ resolveRef[NamedExpression](ref, query)
+ case _ =>
+ throw new AnalysisException(s"$expr is not currently supported")
+ }
+ }
+
+ private def toCatalyst(direction: V2SortDirection): SortDirection = direction match {
+ case V2SortDirection.ASCENDING => Ascending
+ case V2SortDirection.DESCENDING => Descending
+ }
+
+ private def toCatalyst(nullOrdering: V2NullOrdering): NullOrdering = nullOrdering match {
+ case V2NullOrdering.NULLS_FIRST => NullsFirst
+ case V2NullOrdering.NULLS_LAST => NullsLast
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
new file mode 100644
index 000000000..4d369ca07
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions.And
+import org.apache.spark.sql.catalyst.expressions.CaseWhen
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.If
+import org.apache.spark.sql.catalyst.expressions.In
+import org.apache.spark.sql.catalyst.expressions.InSet
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
+import org.apache.spark.sql.catalyst.expressions.Not
+import org.apache.spark.sql.catalyst.expressions.Or
+import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
+import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.InsertAction
+import org.apache.spark.sql.catalyst.plans.logical.InsertStarAction
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeAction
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
+import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateStarAction
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.INSET
+import org.apache.spark.sql.catalyst.trees.TreePattern.NULL_LITERAL
+import org.apache.spark.sql.catalyst.trees.TreePattern.TRUE_OR_FALSE_LITERAL
+import org.apache.spark.sql.types.BooleanType
+import org.apache.spark.util.Utils
+
+/**
+ * A rule similar to ReplaceNullWithFalseInPredicate in Spark but applies to Iceberg row-level commands.
+ */
+object ExtendedReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
+
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+ _.containsAnyPattern(NULL_LITERAL, TRUE_OR_FALSE_LITERAL, INSET)) {
+
+ case d @ DeleteFromIcebergTable(_, Some(cond), _) =>
+ d.copy(condition = Some(replaceNullWithFalse(cond)))
+
+ case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
+ u.copy(condition = Some(replaceNullWithFalse(cond)))
+
+ case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
+ m.copy(
+ mergeCondition = replaceNullWithFalse(mergeCond),
+ matchedActions = replaceNullWithFalse(matchedActions),
+ notMatchedActions = replaceNullWithFalse(notMatchedActions))
+ }
+
+ /**
+ * Recursively traverse the Boolean-type expression to replace
+ * `Literal(null, BooleanType)` with `FalseLiteral`, if possible.
+ *
+ * Note that `transformExpressionsDown` can not be used here as we must stop as soon as we hit
+ * an expression that is not [[CaseWhen]], [[If]], [[And]], [[Or]] or
+ * `Literal(null, BooleanType)`.
+ */
+ private def replaceNullWithFalse(e: Expression): Expression = e match {
+ case Literal(null, BooleanType) =>
+ FalseLiteral
+ // In SQL, the `Not(IN)` expression evaluates as follows:
+ // `NULL not in (1)` -> NULL
+ // `NULL not in (1, NULL)` -> NULL
+ // `1 not in (1, NULL)` -> false
+ // `1 not in (2, NULL)` -> NULL
+ // In predicate, NULL is equal to false, so we can simplify them to false directly.
+ case Not(In(value, list)) if (value +: list).exists(isNullLiteral) =>
+ FalseLiteral
+ case Not(InSet(value, list)) if isNullLiteral(value) || list.contains(null) =>
+ FalseLiteral
+
+ case And(left, right) =>
+ And(replaceNullWithFalse(left), replaceNullWithFalse(right))
+ case Or(left, right) =>
+ Or(replaceNullWithFalse(left), replaceNullWithFalse(right))
+ case cw: CaseWhen if cw.dataType == BooleanType =>
+ val newBranches = cw.branches.map { case (cond, value) =>
+ replaceNullWithFalse(cond) -> replaceNullWithFalse(value)
+ }
+ val newElseValue = cw.elseValue.map(replaceNullWithFalse).getOrElse(FalseLiteral)
+ CaseWhen(newBranches, newElseValue)
+ case i @ If(pred, trueVal, falseVal) if i.dataType == BooleanType =>
+ If(replaceNullWithFalse(pred), replaceNullWithFalse(trueVal), replaceNullWithFalse(falseVal))
+ case e if e.dataType == BooleanType =>
+ e
+ case e =>
+ val message = "Expected a Boolean type expression in replaceNullWithFalse, " +
+ s"but got the type `${e.dataType.catalogString}` in `${e.sql}`."
+ if (Utils.isTesting) {
+ throw new IllegalArgumentException(message)
+ } else {
+ logWarning(message)
+ e
+ }
+ }
+
+ private def isNullLiteral(e: Expression): Boolean = e match {
+ case Literal(null, _) => true
+ case _ => false
+ }
+
+ private def replaceNullWithFalse(mergeActions: Seq[MergeAction]): Seq[MergeAction] = {
+ mergeActions.map {
+ case u @ UpdateAction(Some(cond), _) => u.copy(condition = Some(replaceNullWithFalse(cond)))
+ case u @ UpdateStarAction(Some(cond)) => u.copy(condition = Some(replaceNullWithFalse(cond)))
+ case d @ DeleteAction(Some(cond)) => d.copy(condition = Some(replaceNullWithFalse(cond)))
+ case i @ InsertAction(Some(cond), _) => i.copy(condition = Some(replaceNullWithFalse(cond)))
+ case i @ InsertStarAction(Some(cond)) => i.copy(condition = Some(replaceNullWithFalse(cond)))
+ case other => other
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
new file mode 100644
index 000000000..74cf922c4
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.expressions.And
+import org.apache.spark.sql.catalyst.expressions.CaseWhen
+import org.apache.spark.sql.catalyst.expressions.Coalesce
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.If
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
+import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
+import org.apache.spark.sql.catalyst.expressions.Not
+import org.apache.spark.sql.catalyst.expressions.Or
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.CASE_WHEN
+import org.apache.spark.sql.catalyst.trees.TreePattern.IF
+import org.apache.spark.sql.types.BooleanType
+
+/**
+ * A rule similar to SimplifyConditionalsInPredicate in Spark but applies to Iceberg row-level commands.
+ */
+object ExtendedSimplifyConditionalsInPredicate extends Rule[LogicalPlan] {
+
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+ _.containsAnyPattern(CASE_WHEN, IF)) {
+
+ case d @ DeleteFromIcebergTable(_, Some(cond), _) =>
+ d.copy(condition = Some(simplifyConditional(cond)))
+
+ case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
+ u.copy(condition = Some(simplifyConditional(cond)))
+
+ case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
+ m.copy(
+ mergeCondition = simplifyConditional(mergeCond),
+ matchedActions = simplifyConditional(matchedActions),
+ notMatchedActions = simplifyConditional(notMatchedActions))
+ }
+
+ private def simplifyConditional(e: Expression): Expression = e match {
+ case And(left, right) => And(simplifyConditional(left), simplifyConditional(right))
+ case Or(left, right) => Or(simplifyConditional(left), simplifyConditional(right))
+ case If(cond, trueValue, FalseLiteral) => And(cond, trueValue)
+ case If(cond, trueValue, TrueLiteral) => Or(Not(Coalesce(Seq(cond, FalseLiteral))), trueValue)
+ case If(cond, FalseLiteral, falseValue) =>
+ And(Not(Coalesce(Seq(cond, FalseLiteral))), falseValue)
+ case If(cond, TrueLiteral, falseValue) => Or(cond, falseValue)
+ case CaseWhen(Seq((cond, trueValue)),
+ Some(FalseLiteral) | Some(Literal(null, BooleanType)) | None) =>
+ And(cond, trueValue)
+ case CaseWhen(Seq((cond, trueValue)), Some(TrueLiteral)) =>
+ Or(Not(Coalesce(Seq(cond, FalseLiteral))), trueValue)
+ case CaseWhen(Seq((cond, FalseLiteral)), Some(elseValue)) =>
+ And(Not(Coalesce(Seq(cond, FalseLiteral))), elseValue)
+ case CaseWhen(Seq((cond, TrueLiteral)), Some(elseValue)) =>
+ Or(cond, elseValue)
+ case e if e.dataType == BooleanType => e
+ case e =>
+ assert(e.dataType != BooleanType,
+ "Expected a Boolean type expression in ExtendedSimplifyConditionalsInPredicate, " +
+ s"but got the type `${e.dataType.catalogString}` in `${e.sql}`.")
+ e
+ }
+
+ private def simplifyConditional(mergeActions: Seq[MergeAction]): Seq[MergeAction] = {
+ mergeActions.map {
+ case u @ UpdateAction(Some(cond), _) => u.copy(condition = Some(simplifyConditional(cond)))
+ case u @ UpdateStarAction(Some(cond)) => u.copy(condition = Some(simplifyConditional(cond)))
+ case d @ DeleteAction(Some(cond)) => d.copy(condition = Some(simplifyConditional(cond)))
+ case i @ InsertAction(Some(cond), _) => i.copy(condition = Some(simplifyConditional(cond)))
+ case i @ InsertStarAction(Some(cond)) => i.copy(condition = Some(simplifyConditional(cond)))
+ case other => other
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
new file mode 100644
index 000000000..0339d8bff
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser.extensions
+
+import java.util.Locale
+import org.antlr.v4.runtime._
+import org.antlr.v4.runtime.atn.PredictionMode
+import org.antlr.v4.runtime.misc.Interval
+import org.antlr.v4.runtime.misc.ParseCancellationException
+import org.antlr.v4.runtime.tree.TerminalNodeImpl
+import org.apache.iceberg.common.DynConstructors
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
+import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext
+import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoContext
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoTable
+import org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.UpdateTable
+import org.apache.spark.sql.catalyst.trees.Origin
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.execution.command.ExplainCommand
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.VariableSubstitution
+import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types.StructType
+import scala.jdk.CollectionConverters._
+import scala.util.Try
+
+class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserInterface {
+
+ import IcebergSparkSqlExtensionsParser._
+
+ private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get)
+ private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate)
+
+ /**
+ * Parse a string to a DataType.
+ */
+ override def parseDataType(sqlText: String): DataType = {
+ delegate.parseDataType(sqlText)
+ }
+
+ /**
+ * Parse a string to a raw DataType without CHAR/VARCHAR replacement.
+ */
+ def parseRawDataType(sqlText: String): DataType = throw new UnsupportedOperationException()
+
+ /**
+ * Parse a string to an Expression.
+ */
+ override def parseExpression(sqlText: String): Expression = {
+ delegate.parseExpression(sqlText)
+ }
+
+ /**
+ * Parse a string to a TableIdentifier.
+ */
+ override def parseTableIdentifier(sqlText: String): TableIdentifier = {
+ delegate.parseTableIdentifier(sqlText)
+ }
+
+ /**
+ * Parse a string to a FunctionIdentifier.
+ */
+ override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = {
+ delegate.parseFunctionIdentifier(sqlText)
+ }
+
+ /**
+ * Parse a string to a multi-part identifier.
+ */
+ override def parseMultipartIdentifier(sqlText: String): Seq[String] = {
+ delegate.parseMultipartIdentifier(sqlText)
+ }
+
+ /**
+ * Creates StructType for a given SQL string, which is a comma separated list of field
+ * definitions which will preserve the correct Hive metadata.
+ */
+ override def parseTableSchema(sqlText: String): StructType = {
+ delegate.parseTableSchema(sqlText)
+ }
+
+ /**
+ * Parse a string to a LogicalPlan.
+ */
+ override def parsePlan(sqlText: String): LogicalPlan = {
+ val sqlTextAfterSubstitution = substitutor.substitute(sqlText)
+ if (isIcebergCommand(sqlTextAfterSubstitution)) {
+ parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }.asInstanceOf[LogicalPlan]
+ } else {
+ val parsedPlan = delegate.parsePlan(sqlText)
+ parsedPlan match {
+ case e: ExplainCommand =>
+ e.copy(logicalPlan = replaceRowLevelCommands(e.logicalPlan))
+ case p =>
+ replaceRowLevelCommands(p)
+ }
+ }
+ }
+
+ private def replaceRowLevelCommands(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
+ case DeleteFromTable(UnresolvedIcebergTable(aliasedTable), condition) =>
+ DeleteFromIcebergTable(aliasedTable, condition)
+
+ case UpdateTable(UnresolvedIcebergTable(aliasedTable), assignments, condition) =>
+ UpdateIcebergTable(aliasedTable, assignments, condition)
+
+ case MergeIntoTable(UnresolvedIcebergTable(aliasedTable), source, cond, matchedActions, notMatchedActions) =>
+ // cannot construct MergeIntoIcebergTable right away as MERGE operations require special resolution
+ // that's why the condition and actions must be hidden from the regular resolution rules in Spark
+ // see ResolveMergeIntoTableReferences for details
+ val context = MergeIntoContext(cond, matchedActions, notMatchedActions)
+ UnresolvedMergeIntoIcebergTable(aliasedTable, source, context)
+ }
+
+ object UnresolvedIcebergTable {
+
+ def unapply(plan: LogicalPlan): Option[LogicalPlan] = {
+ EliminateSubqueryAliases(plan) match {
+ case UnresolvedRelation(multipartIdentifier, _, _) if isIcebergTable(multipartIdentifier) =>
+ Some(plan)
+ case _ =>
+ None
+ }
+ }
+
+ private def isIcebergTable(multipartIdent: Seq[String]): Boolean = {
+ val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(SparkSession.active, multipartIdent.asJava)
+ catalogAndIdentifier.catalog match {
+ case tableCatalog: TableCatalog =>
+ Try(tableCatalog.loadTable(catalogAndIdentifier.identifier))
+ .map(isIcebergTable)
+ .getOrElse(false)
+
+ case _ =>
+ false
+ }
+ }
+
+ private def isIcebergTable(table: Table): Boolean = table match {
+ case _: SparkTable => true
+ case _ => false
+ }
+ }
+
+ private def isIcebergCommand(sqlText: String): Boolean = {
+ val normalized = sqlText.toLowerCase(Locale.ROOT).trim()
+ // Strip simple SQL comments that terminate a line, e.g. comments starting with `--` .
+ .replaceAll("--.*?\\n", " ")
+ // Strip newlines.
+ .replaceAll("\\s+", " ")
+ // Strip comments of the form /* ... */. This must come after stripping newlines so that
+ // comments that span multiple lines are caught.
+ .replaceAll("/\\*.*?\\*/", " ")
+ .trim()
+ normalized.startsWith("call") || (
+ normalized.startsWith("alter table") && (
+ normalized.contains("add partition field") ||
+ normalized.contains("drop partition field") ||
+ normalized.contains("replace partition field") ||
+ normalized.contains("write ordered by") ||
+ normalized.contains("write locally ordered by") ||
+ normalized.contains("write distributed by") ||
+ normalized.contains("write unordered") ||
+ normalized.contains("set identifier fields") ||
+ normalized.contains("drop identifier fields")))
+ }
+
+ protected def parse[T](command: String)(toResult: IcebergSqlExtensionsParser => T): T = {
+ val lexer = new IcebergSqlExtensionsLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
+ lexer.removeErrorListeners()
+ lexer.addErrorListener(IcebergParseErrorListener)
+
+ val tokenStream = new CommonTokenStream(lexer)
+ val parser = new IcebergSqlExtensionsParser(tokenStream)
+ parser.addParseListener(IcebergSqlExtensionsPostProcessor)
+ parser.removeErrorListeners()
+ parser.addErrorListener(IcebergParseErrorListener)
+
+ try {
+ try {
+ // first, try parsing with potentially faster SLL mode
+ parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
+ toResult(parser)
+ }
+ catch {
+ case _: ParseCancellationException =>
+ // if we fail, parse with LL mode
+ tokenStream.seek(0) // rewind input stream
+ parser.reset()
+
+ // Try Again.
+ parser.getInterpreter.setPredictionMode(PredictionMode.LL)
+ toResult(parser)
+ }
+ }
+ catch {
+ case e: IcebergParseException if e.command.isDefined =>
+ throw e
+ case e: IcebergParseException =>
+ throw e.withCommand(command)
+ case e: AnalysisException =>
+ val position = Origin(e.line, e.startPosition)
+ throw new IcebergParseException(Option(command), e.message, position, position)
+ }
+ }
+}
+
+object IcebergSparkSqlExtensionsParser {
+ private val substitutorCtor: DynConstructors.Ctor[VariableSubstitution] =
+ DynConstructors.builder()
+ .impl(classOf[VariableSubstitution])
+ .impl(classOf[VariableSubstitution], classOf[SQLConf])
+ .build()
+}
+
+/* Copied from Apache Spark's to avoid dependency on Spark Internals */
+class UpperCaseCharStream(wrapped: CodePointCharStream) extends CharStream {
+ override def consume(): Unit = wrapped.consume
+ override def getSourceName(): String = wrapped.getSourceName
+ override def index(): Int = wrapped.index
+ override def mark(): Int = wrapped.mark
+ override def release(marker: Int): Unit = wrapped.release(marker)
+ override def seek(where: Int): Unit = wrapped.seek(where)
+ override def size(): Int = wrapped.size
+
+ override def getText(interval: Interval): String = wrapped.getText(interval)
+
+ // scalastyle:off
+ override def LA(i: Int): Int = {
+ val la = wrapped.LA(i)
+ if (la == 0 || la == IntStream.EOF) la
+ else Character.toUpperCase(la)
+ }
+ // scalastyle:on
+}
+
+/**
+ * The post-processor validates & cleans-up the parse tree during the parse process.
+ */
+case object IcebergSqlExtensionsPostProcessor extends IcebergSqlExtensionsBaseListener {
+
+ /** Remove the back ticks from an Identifier. */
+ override def exitQuotedIdentifier(ctx: QuotedIdentifierContext): Unit = {
+ replaceTokenByIdentifier(ctx, 1) { token =>
+ // Remove the double back ticks in the string.
+ token.setText(token.getText.replace("``", "`"))
+ token
+ }
+ }
+
+ /** Treat non-reserved keywords as Identifiers. */
+ override def exitNonReserved(ctx: NonReservedContext): Unit = {
+ replaceTokenByIdentifier(ctx, 0)(identity)
+ }
+
+ private def replaceTokenByIdentifier(
+ ctx: ParserRuleContext,
+ stripMargins: Int)(
+ f: CommonToken => CommonToken = identity): Unit = {
+ val parent = ctx.getParent
+ parent.removeLastChild()
+ val token = ctx.getChild(0).getPayload.asInstanceOf[Token]
+ val newToken = new CommonToken(
+ new org.antlr.v4.runtime.misc.Pair(token.getTokenSource, token.getInputStream),
+ IcebergSqlExtensionsParser.IDENTIFIER,
+ token.getChannel,
+ token.getStartIndex + stripMargins,
+ token.getStopIndex - stripMargins)
+ parent.addChild(new TerminalNodeImpl(f(newToken)))
+ }
+}
+
+/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
+case object IcebergParseErrorListener extends BaseErrorListener {
+ override def syntaxError(
+ recognizer: Recognizer[_, _],
+ offendingSymbol: scala.Any,
+ line: Int,
+ charPositionInLine: Int,
+ msg: String,
+ e: RecognitionException): Unit = {
+ val (start, stop) = offendingSymbol match {
+ case token: CommonToken =>
+ val start = Origin(Some(line), Some(token.getCharPositionInLine))
+ val length = token.getStopIndex - token.getStartIndex + 1
+ val stop = Origin(Some(line), Some(token.getCharPositionInLine + length))
+ (start, stop)
+ case _ =>
+ val start = Origin(Some(line), Some(charPositionInLine))
+ (start, start)
+ }
+ throw new IcebergParseException(None, msg, start, stop)
+ }
+}
+
+/**
+ * Copied from Apache Spark
+ * A [[ParseException]] is an [[AnalysisException]] that is thrown during the parse process. It
+ * contains fields and an extended error message that make reporting and diagnosing errors easier.
+ */
+class IcebergParseException(
+ val command: Option[String],
+ message: String,
+ val start: Origin,
+ val stop: Origin) extends AnalysisException(message, start.line, start.startPosition) {
+
+ def this(message: String, ctx: ParserRuleContext) = {
+ this(Option(IcebergParserUtils.command(ctx)),
+ message,
+ IcebergParserUtils.position(ctx.getStart),
+ IcebergParserUtils.position(ctx.getStop))
+ }
+
+ override def getMessage: String = {
+ val builder = new StringBuilder
+ builder ++= "\n" ++= message
+ start match {
+ case Origin(Some(l), Some(p)) =>
+ builder ++= s"(line $l, pos $p)\n"
+ command.foreach { cmd =>
+ val (above, below) = cmd.split("\n").splitAt(l)
+ builder ++= "\n== SQL ==\n"
+ above.foreach(builder ++= _ += '\n')
+ builder ++= (0 until p).map(_ => "-").mkString("") ++= "^^^\n"
+ below.foreach(builder ++= _ += '\n')
+ }
+ case _ =>
+ command.foreach { cmd =>
+ builder ++= "\n== SQL ==\n" ++= cmd
+ }
+ }
+ builder.toString
+ }
+
+ def withCommand(cmd: String): IcebergParseException = {
+ new IcebergParseException(Option(cmd), message, start, stop)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
new file mode 100644
index 000000000..8bb4754e5
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.parser.extensions
+
+import org.antlr.v4.runtime._
+import org.antlr.v4.runtime.misc.Interval
+import org.antlr.v4.runtime.tree.ParseTree
+import org.antlr.v4.runtime.tree.TerminalNode
+import org.apache.iceberg.DistributionMode
+import org.apache.iceberg.NullOrder
+import org.apache.iceberg.SortDirection
+import org.apache.iceberg.expressions.Term
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergParserUtils.withOrigin
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser._
+import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.CallArgument
+import org.apache.spark.sql.catalyst.plans.logical.CallStatement
+import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.NamedArgument
+import org.apache.spark.sql.catalyst.plans.logical.PositionalArgument
+import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
+import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+import org.apache.spark.sql.catalyst.trees.Origin
+import org.apache.spark.sql.connector.expressions
+import org.apache.spark.sql.connector.expressions.ApplyTransform
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.IdentityTransform
+import org.apache.spark.sql.connector.expressions.LiteralValue
+import org.apache.spark.sql.connector.expressions.Transform
+import scala.jdk.CollectionConverters._
+
+class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
+
+ private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] = list.asScala
+ private def toSeq[T](list: java.util.List[T]): Seq[T] = toBuffer(list).toSeq
+
+ /**
+ * Create a [[CallStatement]] for a stored procedure call.
+ */
+ override def visitCall(ctx: CallContext): CallStatement = withOrigin(ctx) {
+ val name = toSeq(ctx.multipartIdentifier.parts).map(_.getText)
+ val args = toSeq(ctx.callArgument).map(typedVisit[CallArgument])
+ CallStatement(name, args)
+ }
+
+ /**
+ * Create an ADD PARTITION FIELD logical command.
+ */
+ override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField = withOrigin(ctx) {
+ AddPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform),
+ Option(ctx.name).map(_.getText))
+ }
+
+ /**
+ * Create a DROP PARTITION FIELD logical command.
+ */
+ override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField = withOrigin(ctx) {
+ DropPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform))
+ }
+
+
+ /**
+ * Create an REPLACE PARTITION FIELD logical command.
+ */
+ override def visitReplacePartitionField(ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
+ ReplacePartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform(0)),
+ typedVisit[Transform](ctx.transform(1)),
+ Option(ctx.name).map(_.getText))
+ }
+
+ /**
+ * Create an SET IDENTIFIER FIELDS logical command.
+ */
+ override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields = withOrigin(ctx) {
+ SetIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
+
+ /**
+ * Create an DROP IDENTIFIER FIELDS logical command.
+ */
+ override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields = withOrigin(ctx) {
+ DropIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
+
+ /**
+ * Create a [[SetWriteDistributionAndOrdering]] for changing the write distribution and ordering.
+ */
+ override def visitSetWriteDistributionAndOrdering(
+ ctx: SetWriteDistributionAndOrderingContext): SetWriteDistributionAndOrdering = {
+
+ val tableName = typedVisit[Seq[String]](ctx.multipartIdentifier)
+
+ val (distributionSpec, orderingSpec) = toDistributionAndOrderingSpec(ctx.writeSpec)
+
+ if (distributionSpec == null && orderingSpec == null) {
+ throw new AnalysisException(
+ "ALTER TABLE has no changes: missing both distribution and ordering clauses")
+ }
+
+ val distributionMode = if (distributionSpec != null) {
+ DistributionMode.HASH
+ } else if (orderingSpec.UNORDERED != null || orderingSpec.LOCALLY != null) {
+ DistributionMode.NONE
+ } else {
+ DistributionMode.RANGE
+ }
+
+ val ordering = if (orderingSpec != null && orderingSpec.order != null) {
+ toSeq(orderingSpec.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
+ } else {
+ Seq.empty
+ }
+
+ SetWriteDistributionAndOrdering(tableName, distributionMode, ordering)
+ }
+
+ private def toDistributionAndOrderingSpec(
+ writeSpec: WriteSpecContext): (WriteDistributionSpecContext, WriteOrderingSpecContext) = {
+
+ if (writeSpec.writeDistributionSpec.size > 1) {
+ throw new AnalysisException("ALTER TABLE contains multiple distribution clauses")
+ }
+
+ if (writeSpec.writeOrderingSpec.size > 1) {
+ throw new AnalysisException("ALTER TABLE contains multiple ordering clauses")
+ }
+
+ val distributionSpec = toBuffer(writeSpec.writeDistributionSpec).headOption.orNull
+ val orderingSpec = toBuffer(writeSpec.writeOrderingSpec).headOption.orNull
+
+ (distributionSpec, orderingSpec)
+ }
+
+ /**
+ * Create an order field.
+ */
+ override def visitOrderField(ctx: OrderFieldContext): (Term, SortDirection, NullOrder) = {
+ val term = Spark3Util.toIcebergTerm(typedVisit[Transform](ctx.transform))
+ val direction = Option(ctx.ASC).map(_ => SortDirection.ASC)
+ .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
+ .getOrElse(SortDirection.ASC)
+ val nullOrder = Option(ctx.FIRST).map(_ => NullOrder.NULLS_FIRST)
+ .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
+ .getOrElse(if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
+ (term, direction, nullOrder)
+ }
+
+ /**
+ * Create an IdentityTransform for a column reference.
+ */
+ override def visitIdentityTransform(ctx: IdentityTransformContext): Transform = withOrigin(ctx) {
+ IdentityTransform(FieldReference(typedVisit[Seq[String]](ctx.multipartIdentifier())))
+ }
+
+ /**
+ * Create a named Transform from argument expressions.
+ */
+ override def visitApplyTransform(ctx: ApplyTransformContext): Transform = withOrigin(ctx) {
+ val args = toSeq(ctx.arguments).map(typedVisit[expressions.Expression])
+ ApplyTransform(ctx.transformName.getText, args)
+ }
+
+ /**
+ * Create a transform argument from a column reference or a constant.
+ */
+ override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression = withOrigin(ctx) {
+ val reference = Option(ctx.multipartIdentifier())
+ .map(typedVisit[Seq[String]])
+ .map(FieldReference(_))
+ val literal = Option(ctx.constant)
+ .map(visitConstant)
+ .map(lit => LiteralValue(lit.value, lit.dataType))
+ reference.orElse(literal)
+ .getOrElse(throw new IcebergParseException(s"Invalid transform argument", ctx))
+ }
+
+ /**
+ * Return a multi-part identifier as Seq[String].
+ */
+ override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) {
+ toSeq(ctx.parts).map(_.getText)
+ }
+
+ /**
+ * Create a positional argument in a stored procedure call.
+ */
+ override def visitPositionalArgument(ctx: PositionalArgumentContext): CallArgument = withOrigin(ctx) {
+ val expr = typedVisit[Expression](ctx.expression)
+ PositionalArgument(expr)
+ }
+
+ /**
+ * Create a named argument in a stored procedure call.
+ */
+ override def visitNamedArgument(ctx: NamedArgumentContext): CallArgument = withOrigin(ctx) {
+ val name = ctx.identifier.getText
+ val expr = typedVisit[Expression](ctx.expression)
+ NamedArgument(name, expr)
+ }
+
+ override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) {
+ visit(ctx.statement).asInstanceOf[LogicalPlan]
+ }
+
+ def visitConstant(ctx: ConstantContext): Literal = {
+ delegate.parseExpression(ctx.getText).asInstanceOf[Literal]
+ }
+
+ override def visitExpression(ctx: ExpressionContext): Expression = {
+ // reconstruct the SQL string and parse it using the main Spark parser
+ // while we can avoid the logic to build Spark expressions, we still have to parse them
+ // we cannot call ctx.getText directly since it will not render spaces correctly
+ // that's why we need to recurse down the tree in reconstructSqlString
+ val sqlString = reconstructSqlString(ctx)
+ delegate.parseExpression(sqlString)
+ }
+
+ private def reconstructSqlString(ctx: ParserRuleContext): String = {
+ toBuffer(ctx.children).map {
+ case c: ParserRuleContext => reconstructSqlString(c)
+ case t: TerminalNode => t.getText
+ }.mkString(" ")
+ }
+
+ private def typedVisit[T](ctx: ParseTree): T = {
+ ctx.accept(this).asInstanceOf[T]
+ }
+}
+
+/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
+object IcebergParserUtils {
+
+ private[sql] def withOrigin[T](ctx: ParserRuleContext)(f: => T): T = {
+ val current = CurrentOrigin.get
+ CurrentOrigin.set(position(ctx.getStart))
+ try {
+ f
+ } finally {
+ CurrentOrigin.set(current)
+ }
+ }
+
+ private[sql] def position(token: Token): Origin = {
+ val opt = Option(token)
+ Origin(opt.map(_.getLine), opt.map(_.getCharPositionInLine))
+ }
+
+ /** Get the command which created the token. */
+ private[sql] def command(ctx: ParserRuleContext): String = {
+ val stream = ctx.getStart.getInputStream
+ stream.getText(Interval.of(0, stream.size() - 1))
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
new file mode 100644
index 000000000..4cc7a7bf2
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.planning
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.plans.logical.RowLevelCommand
+import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
+
+/**
+ * An extractor for operations such as DELETE and MERGE that require rewriting data.
+ *
+ * This class extracts the following entities:
+ * - the row-level command (such as DeleteFromIcebergTable);
+ * - the read relation in the rewrite plan that can be either DataSourceV2Relation or
+ * DataSourceV2ScanRelation depending on whether the planning has already happened;
+ * - the current rewrite plan.
+ */
+object RewrittenRowLevelCommand {
+ type ReturnType = (RowLevelCommand, LogicalPlan, LogicalPlan)
+
+ def unapply(plan: LogicalPlan): Option[ReturnType] = plan match {
+ case c: RowLevelCommand if c.rewritePlan.nonEmpty =>
+ val rewritePlan = c.rewritePlan.get
+
+ // both ReplaceData and WriteDelta reference a write relation
+ // but the corresponding read relation should be at the bottom of the write plan
+ // both the write and read relations will share the same RowLevelOperationTable object
+ // that's why it is safe to use reference equality to find the needed read relation
+
+ val allowScanDuplication = c match {
+ // group-based updates that rely on the union approach may have multiple identical scans
+ case _: UpdateIcebergTable if rewritePlan.isInstanceOf[ReplaceData] => true
+ case _ => false
+ }
+
+ rewritePlan match {
+ case rd @ ReplaceData(DataSourceV2Relation(table, _, _, _, _), query, _, _) =>
+ val readRelation = findReadRelation(table, query, allowScanDuplication)
+ readRelation.map((c, _, rd))
+ case wd @ WriteDelta(DataSourceV2Relation(table, _, _, _, _), query, _, _, _) =>
+ val readRelation = findReadRelation(table, query, allowScanDuplication)
+ readRelation.map((c, _, wd))
+ case _ =>
+ None
+ }
+
+ case _ =>
+ None
+ }
+
+ private def findReadRelation(
+ table: Table,
+ plan: LogicalPlan,
+ allowScanDuplication: Boolean): Option[LogicalPlan] = {
+
+ val readRelations = plan.collect {
+ case r: DataSourceV2Relation if r.table eq table => r
+ case r: DataSourceV2ScanRelation if r.relation.table eq table => r
+ }
+
+ // in some cases, the optimizer replaces the v2 read relation with a local relation
+ // for example, there is no reason to query the table if the condition is always false
+ // that's why it is valid not to find the corresponding v2 read relation
+
+ readRelations match {
+ case relations if relations.isEmpty =>
+ None
+
+ case Seq(relation) =>
+ Some(relation)
+
+ case Seq(relation1: DataSourceV2Relation, relation2: DataSourceV2Relation)
+ if allowScanDuplication && (relation1.table eq relation2.table) =>
+ Some(relation1)
+
+ case Seq(relation1: DataSourceV2ScanRelation, relation2: DataSourceV2ScanRelation)
+ if allowScanDuplication && (relation1.scan eq relation2.scan) =>
+ Some(relation1)
+
+ case Seq(relation1, relation2) if allowScanDuplication =>
+ throw new AnalysisException(s"Row-level read relations don't match: $relation1, $relation2")
+
+ case relations if allowScanDuplication =>
+ throw new AnalysisException(s"Expected up to two row-level read relations: $relations")
+
+ case relations =>
+ throw new AnalysisException(s"Expected only one row-level read relation: $relations")
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
new file mode 100644
index 000000000..e8b1b2941
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String]) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"AddPartitionField ${table.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
new file mode 100644
index 000000000..9616dae5a
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.connector.iceberg.catalog.Procedure
+
+case class Call(procedure: Procedure, args: Seq[Expression]) extends LeafCommand {
+ override lazy val output: Seq[Attribute] = procedure.outputType.toAttributes
+
+ override def simpleString(maxFields: Int): String = {
+ s"Call${truncatedString(output.toSeq, "[", ", ", "]", maxFields)} ${procedure.description}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromIcebergTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromIcebergTable.scala
new file mode 100644
index 000000000..d1268e416
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromIcebergTable.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+case class DeleteFromIcebergTable(
+ table: LogicalPlan,
+ condition: Option[Expression],
+ rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
+
+ override def children: Seq[LogicalPlan] = if (rewritePlan.isDefined) {
+ table :: rewritePlan.get :: Nil
+ } else {
+ table :: Nil
+ }
+
+ override def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand = {
+ copy(rewritePlan = Some(newRewritePlan))
+ }
+
+ override protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[LogicalPlan]): DeleteFromIcebergTable = {
+ if (newChildren.size == 1) {
+ copy(table = newChildren.head, rewritePlan = None)
+ } else {
+ require(newChildren.size == 2, "DeleteFromIcebergTable expects either one or two children")
+ val Seq(newTable, newRewritePlan) = newChildren.take(2)
+ copy(table = newTable, rewritePlan = Some(newRewritePlan))
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
new file mode 100644
index 000000000..29dd686a0
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class DropIdentifierFields(
+ table: Seq[String],
+ fields: Seq[String]) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropIdentifierFields ${table.quoted} (${fields.quoted})"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
new file mode 100644
index 000000000..fb1451324
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class DropPartitionField(table: Seq[String], transform: Transform) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropPartitionField ${table.quoted} ${transform.describe}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
new file mode 100644
index 000000000..8f84851dc
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+case class MergeIntoIcebergTable(
+ targetTable: LogicalPlan,
+ sourceTable: LogicalPlan,
+ mergeCondition: Expression,
+ matchedActions: Seq[MergeAction],
+ notMatchedActions: Seq[MergeAction],
+ rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
+
+ lazy val aligned: Boolean = {
+ val matchedActionsAligned = matchedActions.forall {
+ case UpdateAction(_, assignments) =>
+ AssignmentUtils.aligned(targetTable, assignments)
+ case _: DeleteAction =>
+ true
+ case _ =>
+ false
+ }
+
+ val notMatchedActionsAligned = notMatchedActions.forall {
+ case InsertAction(_, assignments) =>
+ AssignmentUtils.aligned(targetTable, assignments)
+ case _ =>
+ false
+ }
+
+ matchedActionsAligned && notMatchedActionsAligned
+ }
+
+ def condition: Option[Expression] = Some(mergeCondition)
+
+ override def children: Seq[LogicalPlan] = if (rewritePlan.isDefined) {
+ targetTable :: sourceTable :: rewritePlan.get :: Nil
+ } else {
+ targetTable :: sourceTable :: Nil
+ }
+
+ override def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand = {
+ copy(rewritePlan = Some(newRewritePlan))
+ }
+
+ override protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[LogicalPlan]): MergeIntoIcebergTable = {
+
+ newChildren match {
+ case Seq(newTarget, newSource) =>
+ copy(targetTable = newTarget, sourceTable = newSource, rewritePlan = None)
+ case Seq(newTarget, newSource, newRewritePlan) =>
+ copy(targetTable = newTarget, sourceTable = newSource, rewritePlan = Some(newRewritePlan))
+ case _ =>
+ throw new IllegalArgumentException("MergeIntoIcebergTable expects either two or three children")
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
new file mode 100644
index 000000000..3607194fe
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.util.truncatedString
+
+case class MergeRows(
+ isSourceRowPresent: Expression,
+ isTargetRowPresent: Expression,
+ matchedConditions: Seq[Expression],
+ matchedOutputs: Seq[Seq[Expression]],
+ notMatchedConditions: Seq[Expression],
+ notMatchedOutputs: Seq[Seq[Expression]],
+ targetOutput: Seq[Expression],
+ rowIdAttrs: Seq[Attribute],
+ performCardinalityCheck: Boolean,
+ emitNotMatchedTargetRows: Boolean,
+ output: Seq[Attribute],
+ child: LogicalPlan) extends UnaryNode {
+
+ require(targetOutput.nonEmpty || !emitNotMatchedTargetRows)
+
+ override lazy val producedAttributes: AttributeSet = {
+ AttributeSet(output.filterNot(attr => inputSet.contains(attr)))
+ }
+
+ override lazy val references: AttributeSet = child.outputSet
+
+ override def simpleString(maxFields: Int): String = {
+ s"MergeRows${truncatedString(output, "[", ", ", "]", maxFields)}"
+ }
+
+ override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = {
+ copy(child = newChild)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala
new file mode 100644
index 000000000..c21df71f0
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class NoStatsUnaryNode(child: LogicalPlan) extends UnaryNode {
+ override def output: Seq[Attribute] = child.output
+ override def stats: Statistics = Statistics(Long.MaxValue)
+
+ override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = {
+ copy(child = newChild)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala
new file mode 100644
index 000000000..3bf726ffb
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.analysis.NamedRelation
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.connector.write.Write
+import org.apache.spark.sql.types.DataType
+
+/**
+ * Replace data in an existing table.
+ */
+case class ReplaceData(
+ table: NamedRelation,
+ query: LogicalPlan,
+ originalTable: NamedRelation,
+ write: Option[Write] = None) extends V2WriteCommandLike {
+
+ override lazy val references: AttributeSet = query.outputSet
+ override lazy val stringArgs: Iterator[Any] = Iterator(table, query, write)
+
+ // the incoming query may include metadata columns
+ lazy val dataInput: Seq[Attribute] = {
+ val tableAttrNames = table.output.map(_.name)
+ query.output.filter(attr => tableAttrNames.exists(conf.resolver(_, attr.name)))
+ }
+
+ override def outputResolved: Boolean = {
+ assert(table.resolved && query.resolved,
+ "`outputResolved` can only be called when `table` and `query` are both resolved.")
+
+ // take into account only incoming data columns and ignore metadata columns in the query
+ // they will be discarded after the logical write is built in the optimizer
+ // metadata columns may be needed to request a correct distribution or ordering
+ // but are not passed back to the data source during writes
+
+ table.skipSchemaResolution || (dataInput.size == table.output.size &&
+ dataInput.zip(table.output).forall { case (inAttr, outAttr) =>
+ val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
+ // names and types must match, nullability must be compatible
+ inAttr.name == outAttr.name &&
+ DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outType) &&
+ (outAttr.nullable || !inAttr.nullable)
+ })
+ }
+
+ override protected def withNewChildInternal(newChild: LogicalPlan): ReplaceData = {
+ copy(query = newChild)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
new file mode 100644
index 000000000..8c660c6f3
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class ReplacePartitionField(
+ table: Seq[String],
+ transformFrom: Transform,
+ transformTo: Transform,
+ name: Option[String]) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"ReplacePartitionField ${table.quoted} ${transformFrom.describe} " +
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala
new file mode 100644
index 000000000..837ee963b
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+trait RowLevelCommand extends Command with SupportsSubquery {
+ def condition: Option[Expression]
+ def rewritePlan: Option[LogicalPlan]
+ def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
new file mode 100644
index 000000000..a5fa28a61
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class SetIdentifierFields(
+ table: Seq[String],
+ fields: Seq[String]) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"SetIdentifierFields ${table.quoted} (${fields.quoted})"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala
new file mode 100644
index 000000000..895aa733f
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+/**
+ * A node that hides the MERGE condition and actions from regular Spark resolution.
+ */
+case class UnresolvedMergeIntoIcebergTable(
+ targetTable: LogicalPlan,
+ sourceTable: LogicalPlan,
+ context: MergeIntoContext) extends BinaryCommand {
+
+ def duplicateResolved: Boolean = targetTable.outputSet.intersect(sourceTable.outputSet).isEmpty
+
+ override def left: LogicalPlan = targetTable
+ override def right: LogicalPlan = sourceTable
+
+ override protected def withNewChildrenInternal(newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan = {
+ copy(targetTable = newLeft, sourceTable = newRight)
+ }
+}
+
+case class MergeIntoContext(
+ mergeCondition: Expression,
+ matchedActions: Seq[MergeAction],
+ notMatchedActions: Seq[MergeAction])
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala
new file mode 100644
index 000000000..790eb9380
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+case class UpdateIcebergTable(
+ table: LogicalPlan,
+ assignments: Seq[Assignment],
+ condition: Option[Expression],
+ rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
+
+ lazy val aligned: Boolean = AssignmentUtils.aligned(table, assignments)
+
+ override def children: Seq[LogicalPlan] = if (rewritePlan.isDefined) {
+ table :: rewritePlan.get :: Nil
+ } else {
+ table :: Nil
+ }
+
+ override def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand = {
+ copy(rewritePlan = Some(newRewritePlan))
+ }
+
+ override protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[LogicalPlan]): UpdateIcebergTable = {
+ if (newChildren.size == 1) {
+ copy(table = newChildren.head, rewritePlan = None)
+ } else {
+ require(newChildren.size == 2, "UpdateTable expects either one or two children")
+ val Seq(newTable, newRewritePlan) = newChildren.take(2)
+ copy(table = newTable, rewritePlan = Some(newRewritePlan))
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
new file mode 100644
index 000000000..9192d74b7
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.analysis.NamedRelation
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+
+// a node similar to V2WriteCommand in Spark but does not extend Command
+// as ReplaceData and WriteDelta that extend this trait are nested within other commands
+trait V2WriteCommandLike extends UnaryNode {
+ def table: NamedRelation
+ def query: LogicalPlan
+ def outputResolved: Boolean
+
+ override lazy val resolved: Boolean = table.resolved && query.resolved && outputResolved
+
+ override def child: LogicalPlan = query
+ override def output: Seq[Attribute] = Seq.empty
+ override def producedAttributes: AttributeSet = outputSet
+ // Commands are eagerly executed. They will be converted to LocalRelation after the DataFrame
+ // is created. That said, the statistics of a command is useless. Here we just return a dummy
+ // statistics to avoid unnecessary statistics calculation of command's children.
+ override def stats: Statistics = Statistics.DUMMY
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteDelta.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteDelta.scala
new file mode 100644
index 000000000..534427cc0
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteDelta.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
+import org.apache.spark.sql.catalyst.analysis.NamedRelation
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
+import org.apache.spark.sql.catalyst.expressions.NamedExpression
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.catalyst.util.RowDeltaUtils.OPERATION_COLUMN
+import org.apache.spark.sql.catalyst.util.WriteDeltaProjections
+import org.apache.spark.sql.connector.iceberg.write.DeltaWrite
+import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
+import org.apache.spark.sql.connector.write.RowLevelOperationTable
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
+import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types.IntegerType
+import org.apache.spark.sql.types.StructField
+
+/**
+ * Writes a delta of rows to an existing table.
+ */
+case class WriteDelta(
+ table: NamedRelation,
+ query: LogicalPlan,
+ originalTable: NamedRelation,
+ projections: WriteDeltaProjections,
+ write: Option[DeltaWrite] = None) extends V2WriteCommandLike {
+
+ override protected lazy val stringArgs: Iterator[Any] = Iterator(table, query, write)
+
+ private def operationResolved: Boolean = {
+ val attr = query.output.head
+ attr.name == OPERATION_COLUMN && attr.dataType == IntegerType && !attr.nullable
+ }
+
+ private def operation: SupportsDelta = {
+ EliminateSubqueryAliases(table) match {
+ case DataSourceV2Relation(RowLevelOperationTable(_, operation), _, _, _, _) =>
+ operation match {
+ case supportsDelta: SupportsDelta =>
+ supportsDelta
+ case _ =>
+ throw new AnalysisException(s"Operation $operation is not a delta operation")
+ }
+ case _ =>
+ throw new AnalysisException(s"Cannot retrieve row-level operation from $table")
+ }
+ }
+
+ private def rowAttrsResolved: Boolean = {
+ table.skipSchemaResolution || (projections.rowProjection match {
+ case Some(projection) =>
+ table.output.size == projection.schema.size &&
+ projection.schema.zip(table.output).forall { case (field, outAttr) =>
+ isCompatible(field, outAttr)
+ }
+ case None =>
+ true
+ })
+ }
+
+ private def rowIdAttrsResolved: Boolean = {
+ val rowIdAttrs = ExtendedV2ExpressionUtils.resolveRefs[AttributeReference](
+ operation.rowId.toSeq,
+ originalTable)
+
+ projections.rowIdProjection.schema.forall { field =>
+ rowIdAttrs.exists(rowIdAttr => isCompatible(field, rowIdAttr))
+ }
+ }
+
+ private def metadataAttrsResolved: Boolean = {
+ projections.metadataProjection match {
+ case Some(projection) =>
+ val metadataAttrs = ExtendedV2ExpressionUtils.resolveRefs[AttributeReference](
+ operation.requiredMetadataAttributes.toSeq,
+ originalTable)
+
+ projection.schema.forall { field =>
+ metadataAttrs.exists(metadataAttr => isCompatible(field, metadataAttr))
+ }
+ case None =>
+ true
+ }
+ }
+
+ private def isCompatible(projectionField: StructField, outAttr: NamedExpression): Boolean = {
+ val inType = CharVarcharUtils.getRawType(projectionField.metadata).getOrElse(outAttr.dataType)
+ val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
+ // names and types must match, nullability must be compatible
+ projectionField.name == outAttr.name &&
+ DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
+ (outAttr.nullable || !projectionField.nullable)
+ }
+
+ override def outputResolved: Boolean = {
+ assert(table.resolved && query.resolved,
+ "`outputResolved` can only be called when `table` and `query` are both resolved.")
+
+ operationResolved && rowAttrsResolved && rowIdAttrsResolved && metadataAttrsResolved
+ }
+
+ override protected def withNewChildInternal(newChild: LogicalPlan): WriteDelta = {
+ copy(query = newChild)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
new file mode 100644
index 000000000..be15f32bc
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+/**
+ * A CALL statement, as parsed from SQL.
+ */
+case class CallStatement(name: Seq[String], args: Seq[CallArgument]) extends LeafParsedStatement
+
+/**
+ * An argument in a CALL statement.
+ */
+sealed trait CallArgument {
+ def expr: Expression
+}
+
+/**
+ * An argument in a CALL statement identified by name.
+ */
+case class NamedArgument(name: String, expr: Expression) extends CallArgument
+
+/**
+ * An argument in a CALL statement identified by position.
+ */
+case class PositionalArgument(expr: Expression) extends CallArgument
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala
new file mode 100644
index 000000000..9f8f07b77
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+object RowDeltaUtils {
+ final val OPERATION_COLUMN: String = "__row_operation"
+ final val DELETE_OPERATION: Int = 1
+ final val UPDATE_OPERATION: Int = 2
+ final val INSERT_OPERATION: Int = 3
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala
new file mode 100644
index 000000000..e4e8b147d
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.sql.catalyst.ProjectingInternalRow
+
+case class WriteDeltaProjections(
+ rowProjection: Option[ProjectingInternalRow],
+ rowIdProjection: ProjectingInternalRow,
+ metadataProjection: Option[ProjectingInternalRow])
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/expressions/TruncateTransform.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/expressions/TruncateTransform.scala
new file mode 100644
index 000000000..2a3269e2d
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/expressions/TruncateTransform.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.connector.expressions
+
+import org.apache.spark.sql.types.IntegerType
+
+private[sql] object TruncateTransform {
+ def unapply(expr: Expression): Option[(Int, FieldReference)] = expr match {
+ case transform: Transform =>
+ transform match {
+ case NamedTransform("truncate", Seq(Ref(seq: Seq[String]), Lit(value: Int, IntegerType))) =>
+ Some((value, FieldReference(seq)))
+ case NamedTransform("truncate", Seq(Lit(value: Int, IntegerType), Ref(seq: Seq[String]))) =>
+ Some((value, FieldReference(seq)))
+ case _ =>
+ None
+ }
+ case _ =>
+ None
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/ExtendedLogicalWriteInfoImpl.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/ExtendedLogicalWriteInfoImpl.scala
new file mode 100644
index 000000000..0ae10ada5
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/ExtendedLogicalWriteInfoImpl.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.connector.write
+
+import org.apache.spark.sql.connector.iceberg.write.ExtendedLogicalWriteInfo
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+private[sql] case class ExtendedLogicalWriteInfoImpl(
+ queryId: String,
+ schema: StructType,
+ options: CaseInsensitiveStringMap,
+ rowIdSchema: StructType = null,
+ metadataSchema: StructType = null) extends ExtendedLogicalWriteInfo
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationInfoImpl.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationInfoImpl.scala
new file mode 100644
index 000000000..fca42feae
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationInfoImpl.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.connector.write
+
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation.Command
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperationInfo
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+case class RowLevelOperationInfoImpl(
+ command: Command,
+ options: CaseInsensitiveStringMap) extends RowLevelOperationInfo
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala
new file mode 100644
index 000000000..b6cca0af1
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.connector.write
+
+import java.util
+import org.apache.spark.sql.connector.catalog.SupportsRead
+import org.apache.spark.sql.connector.catalog.SupportsWrite
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.connector.catalog.TableCapability
+import org.apache.spark.sql.connector.iceberg.catalog.SupportsRowLevelOperations
+import org.apache.spark.sql.connector.iceberg.write.ExtendedLogicalWriteInfo
+import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+/**
+ * An internal v2 table implementation that wraps the original table during DELETE, UPDATE,
+ * MERGE operations.
+ */
+case class RowLevelOperationTable(
+ table: Table with SupportsRowLevelOperations,
+ operation: RowLevelOperation) extends Table with SupportsRead with SupportsWrite {
+
+ override def name: String = table.name
+ override def schema: StructType = table.schema
+ override def capabilities: util.Set[TableCapability] = table.capabilities
+ override def toString: String = table.toString
+
+ override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = {
+ operation.newScanBuilder(options)
+ }
+
+ override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+ operation.newWriteBuilder(info.asInstanceOf[ExtendedLogicalWriteInfo])
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
new file mode 100644
index 000000000..55f327f7e
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class AddPartitionFieldExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ transform: Transform,
+ name: Option[String]) extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ iceberg.table.updateSpec()
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
+ .commit()
+
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot add partition field to non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"AddPartitionField ${catalog.name}.${ident.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
new file mode 100644
index 000000000..f66962a8c
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.connector.iceberg.catalog.Procedure
+import scala.collection.compat.immutable.ArraySeq
+
+case class CallExec(
+ output: Seq[Attribute],
+ procedure: Procedure,
+ input: InternalRow) extends LeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ ArraySeq.unsafeWrapArray(procedure.call(input))
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"CallExec${truncatedString(output, "[", ", ", "]", maxFields)} ${procedure.description}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
new file mode 100644
index 000000000..dee778b47
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions
+import org.apache.iceberg.relocated.com.google.common.collect.Sets
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+
+case class DropIdentifierFieldsExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ fields: Seq[String]) extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val schema = iceberg.table.schema
+ val identifierFieldNames = Sets.newHashSet(schema.identifierFieldNames)
+
+ for (name <- fields) {
+ Preconditions.checkArgument(schema.findField(name) != null,
+ "Cannot complete drop identifier fields operation: field %s not found", name)
+ Preconditions.checkArgument(identifierFieldNames.contains(name),
+ "Cannot complete drop identifier fields operation: %s is not an identifier field", name)
+ identifierFieldNames.remove(name)
+ }
+
+ iceberg.table.updateSchema()
+ .setIdentifierFields(identifierFieldNames)
+ .commit();
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot drop identifier fields in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
new file mode 100644
index 000000000..9a153f0c0
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.IdentityTransform
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class DropPartitionFieldExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ transform: Transform) extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val schema = iceberg.table.schema
+ transform match {
+ case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
+ iceberg.table.updateSpec()
+ .removeField(parts.head)
+ .commit()
+
+ case _ =>
+ iceberg.table.updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transform))
+ .commit()
+ }
+
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot drop partition field in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropPartitionField ${catalog.name}.${ident.quoted} ${transform.describe}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala
new file mode 100644
index 000000000..85bda0b08
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.connector.write.RowLevelOperationTable
+
+/**
+ * A class similar to DataSourceV2Implicits in Spark but contains custom implicit helpers.
+ */
+object ExtendedDataSourceV2Implicits {
+ implicit class TableHelper(table: Table) {
+ def asRowLevelOperationTable: RowLevelOperationTable = {
+ table match {
+ case rowLevelOperationTable: RowLevelOperationTable =>
+ rowLevelOperationTable
+ case _ =>
+ throw new AnalysisException(s"Table ${table.name} is not a row-level operation table")
+ }
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
new file mode 100644
index 000000000..c3412232e
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.SparkCatalog
+import org.apache.iceberg.spark.SparkSessionCatalog
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.Strategy
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.PredicateHelper
+import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.Call
+import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeRows
+import org.apache.spark.sql.catalyst.plans.logical.NoStatsUnaryNode
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
+import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
+import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.datasources.DataSourceStrategy
+import scala.jdk.CollectionConverters._
+
+case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy with PredicateHelper {
+
+ import DataSourceV2Implicits._
+
+ override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
+ case c @ Call(procedure, args) =>
+ val input = buildInternalRow(args)
+ CallExec(c.output, procedure, input) :: Nil
+
+ case AddPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform, name) =>
+ AddPartitionFieldExec(catalog, ident, transform, name) :: Nil
+
+ case DropPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform) =>
+ DropPartitionFieldExec(catalog, ident, transform) :: Nil
+
+ case ReplacePartitionField(IcebergCatalogAndIdentifier(catalog, ident), transformFrom, transformTo, name) =>
+ ReplacePartitionFieldExec(catalog, ident, transformFrom, transformTo, name) :: Nil
+
+ case SetIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
+ SetIdentifierFieldsExec(catalog, ident, fields) :: Nil
+
+ case DropIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
+ DropIdentifierFieldsExec(catalog, ident, fields) :: Nil
+
+ case SetWriteDistributionAndOrdering(
+ IcebergCatalogAndIdentifier(catalog, ident), distributionMode, ordering) =>
+ SetWriteDistributionAndOrderingExec(catalog, ident, distributionMode, ordering) :: Nil
+
+ case ReplaceData(_: DataSourceV2Relation, query, r: DataSourceV2Relation, Some(write)) =>
+ // refresh the cache using the original relation
+ ReplaceDataExec(planLater(query), refreshCache(r), write) :: Nil
+
+ case WriteDelta(_: DataSourceV2Relation, query, r: DataSourceV2Relation, projs, Some(write)) =>
+ // refresh the cache using the original relation
+ WriteDeltaExec(planLater(query), refreshCache(r), projs, write) :: Nil
+
+ case MergeRows(isSourceRowPresent, isTargetRowPresent, matchedConditions, matchedOutputs, notMatchedConditions,
+ notMatchedOutputs, targetOutput, rowIdAttrs, performCardinalityCheck, emitNotMatchedTargetRows,
+ output, child) =>
+
+ MergeRowsExec(isSourceRowPresent, isTargetRowPresent, matchedConditions, matchedOutputs, notMatchedConditions,
+ notMatchedOutputs, targetOutput, rowIdAttrs, performCardinalityCheck, emitNotMatchedTargetRows,
+ output, planLater(child)) :: Nil
+
+ case DeleteFromIcebergTable(DataSourceV2ScanRelation(r, _, output), condition, None) =>
+ // the optimizer has already checked that this delete can be handled using a metadata operation
+ val deleteCond = condition.getOrElse(Literal.TrueLiteral)
+ val predicates = splitConjunctivePredicates(deleteCond)
+ val normalizedPredicates = DataSourceStrategy.normalizeExprs(predicates, output)
+ val filters = normalizedPredicates.flatMap { pred =>
+ val filter = DataSourceStrategy.translateFilter(pred, supportNestedPredicatePushdown = true)
+ if (filter.isEmpty) {
+ throw QueryCompilationErrors.cannotTranslateExpressionToSourceFilterError(pred)
+ }
+ filter
+ }.toArray
+ DeleteFromTableExec(r.table.asDeletable, filters, refreshCache(r)) :: Nil
+
+ case NoStatsUnaryNode(child) =>
+ planLater(child) :: Nil
+
+ case _ => Nil
+ }
+
+ private def buildInternalRow(exprs: Seq[Expression]): InternalRow = {
+ val values = new Array[Any](exprs.size)
+ for (index <- exprs.indices) {
+ values(index) = exprs(index).eval()
+ }
+ new GenericInternalRow(values)
+ }
+
+ private def refreshCache(r: DataSourceV2Relation)(): Unit = {
+ spark.sharedState.cacheManager.recacheByPlan(spark, r)
+ }
+
+ private object IcebergCatalogAndIdentifier {
+ def unapply(identifier: Seq[String]): Option[(TableCatalog, Identifier)] = {
+ val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(spark, identifier.asJava)
+ catalogAndIdentifier.catalog match {
+ case icebergCatalog: SparkCatalog =>
+ Some((icebergCatalog, catalogAndIdentifier.identifier))
+ case icebergCatalog: SparkSessionCatalog[_] =>
+ Some((icebergCatalog, catalogAndIdentifier.identifier))
+ case _ =>
+ None
+ }
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDistributionAndOrderingUtils.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDistributionAndOrderingUtils.scala
new file mode 100644
index 000000000..8c37b1b75
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDistributionAndOrderingUtils.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils.toCatalyst
+import org.apache.spark.sql.catalyst.expressions.SortOrder
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.RepartitionByExpression
+import org.apache.spark.sql.catalyst.plans.logical.Sort
+import org.apache.spark.sql.connector.distributions.ClusteredDistribution
+import org.apache.spark.sql.connector.distributions.OrderedDistribution
+import org.apache.spark.sql.connector.distributions.UnspecifiedDistribution
+import org.apache.spark.sql.connector.write.RequiresDistributionAndOrdering
+import org.apache.spark.sql.connector.write.Write
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.internal.SQLConf
+import scala.collection.compat.immutable.ArraySeq
+
+/**
+ * A rule that is inspired by DistributionAndOrderingUtils in Spark but supports Iceberg transforms.
+ *
+ * Note that similarly to the original rule in Spark, it does not let AQE pick the number of shuffle
+ * partitions. See SPARK-34230 for context.
+ */
+object ExtendedDistributionAndOrderingUtils {
+
+ def prepareQuery(write: Write, query: LogicalPlan, conf: SQLConf): LogicalPlan = write match {
+ case write: RequiresDistributionAndOrdering =>
+ val numPartitions = write.requiredNumPartitions()
+ val distribution = write.requiredDistribution match {
+ case d: OrderedDistribution => d.ordering.map(e => toCatalyst(e, query))
+ case d: ClusteredDistribution => d.clustering.map(e => toCatalyst(e, query))
+ case _: UnspecifiedDistribution => Array.empty[Expression]
+ }
+
+ val queryWithDistribution = if (distribution.nonEmpty) {
+ val finalNumPartitions = if (numPartitions > 0) {
+ numPartitions
+ } else {
+ conf.numShufflePartitions
+ }
+ // the conversion to catalyst expressions above produces SortOrder expressions
+ // for OrderedDistribution and generic expressions for ClusteredDistribution
+ // this allows RepartitionByExpression to pick either range or hash partitioning
+ RepartitionByExpression(ArraySeq.unsafeWrapArray(distribution), query, finalNumPartitions)
+ } else if (numPartitions > 0) {
+ throw QueryCompilationErrors.numberOfPartitionsNotAllowedWithUnspecifiedDistributionError()
+ } else {
+ query
+ }
+
+ val ordering = write.requiredOrdering.toSeq
+ .map(e => toCatalyst(e, query))
+ .asInstanceOf[Seq[SortOrder]]
+
+ val queryWithDistributionAndOrdering = if (ordering.nonEmpty) {
+ Sort(ordering, global = false, queryWithDistribution)
+ } else {
+ queryWithDistribution
+ }
+
+ queryWithDistributionAndOrdering
+
+ case _ =>
+ query
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala
new file mode 100644
index 000000000..86cab35cb
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import java.util.UUID
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.PredicateHelper
+import org.apache.spark.sql.catalyst.plans.logical.AppendData
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.OverwriteByExpression
+import org.apache.spark.sql.catalyst.plans.logical.OverwritePartitionsDynamic
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.utils.PlanUtils.isIcebergRelation
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.connector.iceberg.write.DeltaWriteBuilder
+import org.apache.spark.sql.connector.write.ExtendedLogicalWriteInfoImpl
+import org.apache.spark.sql.connector.write.SupportsDynamicOverwrite
+import org.apache.spark.sql.connector.write.SupportsOverwrite
+import org.apache.spark.sql.connector.write.SupportsTruncate
+import org.apache.spark.sql.connector.write.WriteBuilder
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.execution.datasources.DataSourceStrategy
+import org.apache.spark.sql.sources.AlwaysTrue
+import org.apache.spark.sql.sources.Filter
+import org.apache.spark.sql.types.StructType
+
+/**
+ * A rule that is inspired by V2Writes in Spark but supports Iceberg transforms.
+ */
+object ExtendedV2Writes extends Rule[LogicalPlan] with PredicateHelper {
+
+ import DataSourceV2Implicits._
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
+ case a @ AppendData(r: DataSourceV2Relation, query, options, _, None) if isIcebergRelation(r) =>
+ val writeBuilder = newWriteBuilder(r.table, query.schema, options)
+ val write = writeBuilder.build()
+ val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
+ a.copy(write = Some(write), query = newQuery)
+
+ case o @ OverwriteByExpression(r: DataSourceV2Relation, deleteExpr, query, options, _, None)
+ if isIcebergRelation(r) =>
+ // fail if any filter cannot be converted. correctness depends on removing all matching data.
+ val filters = splitConjunctivePredicates(deleteExpr).flatMap { pred =>
+ val filter = DataSourceStrategy.translateFilter(pred, supportNestedPredicatePushdown = true)
+ if (filter.isEmpty) {
+ throw QueryCompilationErrors.cannotTranslateExpressionToSourceFilterError(pred)
+ }
+ filter
+ }.toArray
+
+ val table = r.table
+ val writeBuilder = newWriteBuilder(table, query.schema, options)
+ val write = writeBuilder match {
+ case builder: SupportsTruncate if isTruncate(filters) =>
+ builder.truncate().build()
+ case builder: SupportsOverwrite =>
+ builder.overwrite(filters).build()
+ case _ =>
+ throw QueryExecutionErrors.overwriteTableByUnsupportedExpressionError(table)
+ }
+
+ val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
+ o.copy(write = Some(write), query = newQuery)
+
+ case o @ OverwritePartitionsDynamic(r: DataSourceV2Relation, query, options, _, None)
+ if isIcebergRelation(r) =>
+ val table = r.table
+ val writeBuilder = newWriteBuilder(table, query.schema, options)
+ val write = writeBuilder match {
+ case builder: SupportsDynamicOverwrite =>
+ builder.overwriteDynamicPartitions().build()
+ case _ =>
+ throw QueryExecutionErrors.dynamicPartitionOverwriteUnsupportedByTableError(table)
+ }
+ val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
+ o.copy(write = Some(write), query = newQuery)
+
+ case rd @ ReplaceData(r: DataSourceV2Relation, query, _, None) =>
+ val rowSchema = StructType.fromAttributes(rd.dataInput)
+ val writeBuilder = newWriteBuilder(r.table, rowSchema, Map.empty)
+ val write = writeBuilder.build()
+ val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
+ rd.copy(write = Some(write), query = Project(rd.dataInput, newQuery))
+
+ case wd @ WriteDelta(r: DataSourceV2Relation, query, _, projections, None) =>
+ val rowSchema = projections.rowProjection.map(_.schema).orNull
+ val rowIdSchema = projections.rowIdProjection.schema
+ val metadataSchema = projections.metadataProjection.map(_.schema).orNull
+ val writeBuilder = newWriteBuilder(r.table, rowSchema, Map.empty, rowIdSchema, metadataSchema)
+ writeBuilder match {
+ case builder: DeltaWriteBuilder =>
+ val deltaWrite = builder.build()
+ val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(deltaWrite, query, conf)
+ wd.copy(write = Some(deltaWrite), query = newQuery)
+ case other =>
+ throw new AnalysisException(s"$other is not DeltaWriteBuilder")
+ }
+ }
+
+ private def isTruncate(filters: Array[Filter]): Boolean = {
+ filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
+ }
+
+ private def newWriteBuilder(
+ table: Table,
+ rowSchema: StructType,
+ writeOptions: Map[String, String],
+ rowIdSchema: StructType = null,
+ metadataSchema: StructType = null): WriteBuilder = {
+ val info = ExtendedLogicalWriteInfoImpl(
+ queryId = UUID.randomUUID().toString,
+ rowSchema,
+ writeOptions.asOptions,
+ rowIdSchema,
+ metadataSchema)
+ table.asWritable.newWriteBuilder(info)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala
new file mode 100644
index 000000000..4fbf8a523
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.SparkException
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Ascending
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.catalyst.expressions.BasePredicate
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.SortOrder
+import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
+import org.apache.spark.sql.catalyst.expressions.codegen.GeneratePredicate
+import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.UnaryExecNode
+
+case class MergeRowsExec(
+ isSourceRowPresent: Expression,
+ isTargetRowPresent: Expression,
+ matchedConditions: Seq[Expression],
+ matchedOutputs: Seq[Seq[Expression]],
+ notMatchedConditions: Seq[Expression],
+ notMatchedOutputs: Seq[Seq[Expression]],
+ targetOutput: Seq[Expression],
+ rowIdAttrs: Seq[Attribute],
+ performCardinalityCheck: Boolean,
+ emitNotMatchedTargetRows: Boolean,
+ output: Seq[Attribute],
+ child: SparkPlan) extends UnaryExecNode {
+
+ override def requiredChildOrdering: Seq[Seq[SortOrder]] = {
+ if (performCardinalityCheck) {
+ // request a local sort by the row ID attrs to co-locate matches for the same target row
+ Seq(rowIdAttrs.map(attr => SortOrder(attr, Ascending)))
+ } else {
+ Seq(Nil)
+ }
+ }
+
+ @transient override lazy val producedAttributes: AttributeSet = {
+ AttributeSet(output.filterNot(attr => inputSet.contains(attr)))
+ }
+
+ @transient override lazy val references: AttributeSet = child.outputSet
+
+ override def simpleString(maxFields: Int): String = {
+ s"MergeRowsExec${truncatedString(output, "[", ", ", "]", maxFields)}"
+ }
+
+ override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = {
+ copy(child = newChild)
+ }
+
+ protected override def doExecute(): RDD[InternalRow] = {
+ child.execute().mapPartitions(processPartition)
+ }
+
+ private def createProjection(exprs: Seq[Expression], attrs: Seq[Attribute]): UnsafeProjection = {
+ UnsafeProjection.create(exprs, attrs)
+ }
+
+ private def createPredicate(expr: Expression, attrs: Seq[Attribute]): BasePredicate = {
+ GeneratePredicate.generate(expr, attrs)
+ }
+
+ private def applyProjection(
+ actions: Seq[(BasePredicate, Option[UnsafeProjection])],
+ inputRow: InternalRow): InternalRow = {
+
+ // find the first action where the predicate evaluates to true
+ // if there are overlapping conditions in actions, use the first matching action
+ // in the example below, when id = 5, both actions match but the first one is applied
+ // WHEN MATCHED AND id > 1 AND id < 10 UPDATE *
+ // WHEN MATCHED AND id = 5 OR id = 21 DELETE
+
+ val pair = actions.find {
+ case (predicate, _) => predicate.eval(inputRow)
+ }
+
+ // apply the projection to produce an output row, or return null to suppress this row
+ pair match {
+ case Some((_, Some(projection))) =>
+ projection.apply(inputRow)
+ case _ =>
+ null
+ }
+ }
+
+ private def processPartition(rowIterator: Iterator[InternalRow]): Iterator[InternalRow] = {
+ val inputAttrs = child.output
+
+ val isSourceRowPresentPred = createPredicate(isSourceRowPresent, inputAttrs)
+ val isTargetRowPresentPred = createPredicate(isTargetRowPresent, inputAttrs)
+
+ val matchedPreds = matchedConditions.map(createPredicate(_, inputAttrs))
+ val matchedProjs = matchedOutputs.map {
+ case output if output.nonEmpty => Some(createProjection(output, inputAttrs))
+ case _ => None
+ }
+ val matchedPairs = matchedPreds zip matchedProjs
+
+ val notMatchedPreds = notMatchedConditions.map(createPredicate(_, inputAttrs))
+ val notMatchedProjs = notMatchedOutputs.map {
+ case output if output.nonEmpty => Some(createProjection(output, inputAttrs))
+ case _ => None
+ }
+ val nonMatchedPairs = notMatchedPreds zip notMatchedProjs
+
+ val projectTargetCols = createProjection(targetOutput, inputAttrs)
+ val rowIdProj = createProjection(rowIdAttrs, inputAttrs)
+
+ // This method is responsible for processing a input row to emit the resultant row with an
+ // additional column that indicates whether the row is going to be included in the final
+ // output of merge or not.
+ // 1. Found a target row for which there is no corresponding source row (join condition not met)
+ // - Only project the target columns if we need to output unchanged rows
+ // 2. Found a source row for which there is no corresponding target row (join condition not met)
+ // - Apply the not matched actions (i.e INSERT actions) if non match conditions are met.
+ // 3. Found a source row for which there is a corresponding target row (join condition met)
+ // - Apply the matched actions (i.e DELETE or UPDATE actions) if match conditions are met.
+ def processRow(inputRow: InternalRow): InternalRow = {
+ if (emitNotMatchedTargetRows && !isSourceRowPresentPred.eval(inputRow)) {
+ projectTargetCols.apply(inputRow)
+ } else if (!isTargetRowPresentPred.eval(inputRow)) {
+ applyProjection(nonMatchedPairs, inputRow)
+ } else {
+ applyProjection(matchedPairs, inputRow)
+ }
+ }
+
+ var lastMatchedRowId: InternalRow = null
+
+ def processRowWithCardinalityCheck(inputRow: InternalRow): InternalRow = {
+ val isSourceRowPresent = isSourceRowPresentPred.eval(inputRow)
+ val isTargetRowPresent = isTargetRowPresentPred.eval(inputRow)
+
+ if (isSourceRowPresent && isTargetRowPresent) {
+ val currentRowId = rowIdProj.apply(inputRow)
+ if (currentRowId == lastMatchedRowId) {
+ throw new SparkException(
+ "The ON search condition of the MERGE statement matched a single row from " +
+ "the target table with multiple rows of the source table. This could result " +
+ "in the target row being operated on more than once with an update or delete " +
+ "operation and is not allowed.")
+ }
+ lastMatchedRowId = currentRowId.copy()
+ } else {
+ lastMatchedRowId = null
+ }
+
+ if (emitNotMatchedTargetRows && !isSourceRowPresent) {
+ projectTargetCols.apply(inputRow)
+ } else if (!isTargetRowPresent) {
+ applyProjection(nonMatchedPairs, inputRow)
+ } else {
+ applyProjection(matchedPairs, inputRow)
+ }
+ }
+
+ val processFunc: InternalRow => InternalRow = if (performCardinalityCheck) {
+ processRowWithCardinalityCheck
+ } else {
+ processRow
+ }
+
+ rowIterator
+ .map(processFunc)
+ .filter(row => row != null)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
new file mode 100644
index 000000000..54e66db91
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.PredicateHelper
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.connector.catalog.SupportsDelete
+import org.apache.spark.sql.execution.datasources.DataSourceStrategy
+import org.apache.spark.sql.sources
+import org.slf4j.LoggerFactory
+
+/**
+ * Checks whether a metadata delete is possible and nullifies the rewrite plan if the source can
+ * handle this delete without executing the rewrite plan.
+ *
+ * Note this rule must be run after expression optimization.
+ */
+object OptimizeMetadataOnlyDeleteFromTable extends Rule[LogicalPlan] with PredicateHelper {
+
+ val logger = LoggerFactory.getLogger(OptimizeMetadataOnlyDeleteFromTable.getClass)
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+ case d @ DeleteFromIcebergTable(relation: DataSourceV2Relation, cond, Some(_)) =>
+ val deleteCond = cond.getOrElse(Literal.TrueLiteral)
+ relation.table match {
+ case table: SupportsDelete if !SubqueryExpression.hasSubquery(deleteCond) =>
+ val predicates = splitConjunctivePredicates(deleteCond)
+ val normalizedPredicates = DataSourceStrategy.normalizeExprs(predicates, relation.output)
+ val dataSourceFilters = toDataSourceFilters(normalizedPredicates)
+ val allPredicatesTranslated = normalizedPredicates.size == dataSourceFilters.length
+ if (allPredicatesTranslated && table.canDeleteWhere(dataSourceFilters)) {
+ logger.info(s"Optimizing delete expression: ${dataSourceFilters.mkString(",")} as metadata delete")
+ d.copy(rewritePlan = None)
+ } else {
+ d
+ }
+ case _ =>
+ d
+ }
+ }
+
+ protected def toDataSourceFilters(predicates: Seq[Expression]): Array[sources.Filter] = {
+ predicates.flatMap { p =>
+ val filter = DataSourceStrategy.translateFilter(p, supportNestedPredicatePushdown = true)
+ if (filter.isEmpty) {
+ logWarning(s"Cannot translate expression to source filter: $p")
+ }
+ filter
+ }.toArray
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala
new file mode 100644
index 000000000..26c652469
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.connector.write.Write
+import org.apache.spark.sql.execution.SparkPlan
+
+/**
+ * Physical plan node to replace data in existing tables.
+ */
+case class ReplaceDataExec(
+ query: SparkPlan,
+ refreshCache: () => Unit,
+ write: Write) extends V2ExistingTableWriteExec {
+
+ override lazy val references: AttributeSet = query.outputSet
+ override lazy val stringArgs: Iterator[Any] = Iterator(query, write)
+
+ override protected def withNewChildInternal(newChild: SparkPlan): ReplaceDataExec = {
+ copy(query = newChild)
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
new file mode 100644
index 000000000..fcae0a5de
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.IdentityTransform
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class ReplacePartitionFieldExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ transformFrom: Transform,
+ transformTo: Transform,
+ name: Option[String]) extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val schema = iceberg.table.schema
+ transformFrom match {
+ case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
+ iceberg.table.updateSpec()
+ .removeField(parts.head)
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
+
+ case _ =>
+ iceberg.table.updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transformFrom))
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
+ }
+
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot replace partition field in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ReplacePartitionField ${catalog.name}.${ident.quoted} ${transformFrom.describe} " +
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala
new file mode 100644
index 000000000..414d4c0ec
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.RowLevelCommand
+import org.apache.spark.sql.catalyst.rules.Rule
+
+/**
+ * Replaces operations such as DELETE and MERGE with the corresponding rewrite plans.
+ */
+object ReplaceRewrittenRowLevelCommand extends Rule[LogicalPlan] {
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
+ case c: RowLevelCommand if c.rewritePlan.isDefined =>
+ c.rewritePlan.get
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
new file mode 100644
index 000000000..4e89b9a1c
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.PredicateHelper
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.planning.RewrittenRowLevelCommand
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.execution.datasources.DataSourceStrategy
+import org.apache.spark.sql.sources.Filter
+import org.apache.spark.sql.types.StructType
+
+object RowLevelCommandScanRelationPushDown extends Rule[LogicalPlan] with PredicateHelper {
+ import ExtendedDataSourceV2Implicits._
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
+ // push down the filter from the command condition instead of the filter in the rewrite plan,
+ // which may be negated for copy-on-write operations
+ case RewrittenRowLevelCommand(command, relation: DataSourceV2Relation, rewritePlan) =>
+ val table = relation.table.asRowLevelOperationTable
+ val scanBuilder = table.newScanBuilder(relation.options)
+
+ val (pushedFilters, remainingFilters) = command.condition match {
+ case Some(cond) => pushFilters(cond, scanBuilder, relation.output)
+ case None => (Nil, Nil)
+ }
+
+ val (scan, output) = PushDownUtils.pruneColumns(scanBuilder, relation, relation.output, Nil)
+
+ logInfo(
+ s"""
+ |Pushing operators to ${relation.name}
+ |Pushed filters: ${pushedFilters.mkString(", ")}
+ |Filters that were not pushed: ${remainingFilters.mkString(",")}
+ |Output: ${output.mkString(", ")}
+ """.stripMargin)
+
+ // replace DataSourceV2Relation with DataSourceV2ScanRelation for the row operation table
+ // there may be multiple read relations for UPDATEs that rely on the UNION approach
+ val newRewritePlan = rewritePlan transform {
+ case r: DataSourceV2Relation if r.table eq table =>
+ DataSourceV2ScanRelation(r, scan, toOutputAttrs(scan.readSchema(), r))
+ }
+
+ command.withNewRewritePlan(newRewritePlan)
+ }
+
+ private def pushFilters(
+ cond: Expression,
+ scanBuilder: ScanBuilder,
+ tableAttrs: Seq[AttributeReference]): (Seq[Filter], Seq[Expression]) = {
+
+ val tableAttrSet = AttributeSet(tableAttrs)
+ val filters = splitConjunctivePredicates(cond).filter(_.references.subsetOf(tableAttrSet))
+ val normalizedFilters = DataSourceStrategy.normalizeExprs(filters, tableAttrs)
+ val (_, normalizedFiltersWithoutSubquery) =
+ normalizedFilters.partition(SubqueryExpression.hasSubquery)
+
+ PushDownUtils.pushFilters(scanBuilder, normalizedFiltersWithoutSubquery)
+ }
+
+ private def toOutputAttrs(
+ schema: StructType,
+ relation: DataSourceV2Relation): Seq[AttributeReference] = {
+ val nameToAttr = relation.output.map(_.name).zip(relation.output).toMap
+ val cleaned = CharVarcharUtils.replaceCharVarcharWithStringInSchema(schema)
+ cleaned.toAttributes.map {
+ // keep the attribute id during transformation
+ a => a.withExprId(nameToAttr(a.name).exprId)
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
new file mode 100644
index 000000000..b50550ad3
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import scala.jdk.CollectionConverters._
+
+case class SetIdentifierFieldsExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ fields: Seq[String]) extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ iceberg.table.updateSchema()
+ .setIdentifierFields(fields.asJava)
+ .commit();
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot set identifier fields in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"SetIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
new file mode 100644
index 000000000..386485b10
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.DistributionMode
+import org.apache.iceberg.NullOrder
+import org.apache.iceberg.SortDirection
+import org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE
+import org.apache.iceberg.expressions.Term
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+
+case class SetWriteDistributionAndOrderingExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ distributionMode: DistributionMode,
+ sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafV2CommandExec {
+
+ import CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val txn = iceberg.table.newTransaction()
+
+ val orderBuilder = txn.replaceSortOrder()
+ sortOrder.foreach {
+ case (term, SortDirection.ASC, nullOrder) =>
+ orderBuilder.asc(term, nullOrder)
+ case (term, SortDirection.DESC, nullOrder) =>
+ orderBuilder.desc(term, nullOrder)
+ }
+ orderBuilder.commit()
+
+ txn.updateProperties()
+ .set(WRITE_DISTRIBUTION_MODE, distributionMode.modeName())
+ .commit()
+
+ txn.commitTransaction()
+
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot set write order of non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ val tableIdent = s"${catalog.name}.${ident.quoted}"
+ val order = sortOrder.map {
+ case (term, direction, nullOrder) => s"$term $direction $nullOrder"
+ }.mkString(", ")
+ s"SetWriteDistributionAndOrdering $tableIdent $distributionMode $order"
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteDeltaExec.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteDeltaExec.scala
new file mode 100644
index 000000000..fa4e4f648
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteDeltaExec.scala
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.SparkEnv
+import org.apache.spark.SparkException
+import org.apache.spark.TaskContext
+import org.apache.spark.internal.Logging
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.AttributeSet
+import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
+import org.apache.spark.sql.catalyst.util.WriteDeltaProjections
+import org.apache.spark.sql.connector.iceberg.write.DeltaWrite
+import org.apache.spark.sql.connector.iceberg.write.DeltaWriter
+import org.apache.spark.sql.connector.write.BatchWrite
+import org.apache.spark.sql.connector.write.DataWriter
+import org.apache.spark.sql.connector.write.DataWriterFactory
+import org.apache.spark.sql.connector.write.PhysicalWriteInfoImpl
+import org.apache.spark.sql.connector.write.WriterCommitMessage
+import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.metric.CustomMetrics
+import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.util.LongAccumulator
+import org.apache.spark.util.Utils
+import scala.collection.compat.immutable.ArraySeq
+import scala.util.control.NonFatal
+
+/**
+ * Physical plan node to write a delta of rows to an existing table.
+ */
+case class WriteDeltaExec(
+ query: SparkPlan,
+ refreshCache: () => Unit,
+ projections: WriteDeltaProjections,
+ write: DeltaWrite) extends ExtendedV2ExistingTableWriteExec[DeltaWriter[InternalRow]] {
+
+ override lazy val references: AttributeSet = query.outputSet
+ override lazy val stringArgs: Iterator[Any] = Iterator(query, write)
+
+ override lazy val writingTask: WritingSparkTask[DeltaWriter[InternalRow]] = {
+ DeltaWithMetadataWritingSparkTask(projections)
+ }
+
+ override protected def withNewChildInternal(newChild: SparkPlan): WriteDeltaExec = {
+ copy(query = newChild)
+ }
+}
+
+// a trait similar to V2ExistingTableWriteExec but supports custom write tasks
+trait ExtendedV2ExistingTableWriteExec[W <: DataWriter[InternalRow]] extends V2ExistingTableWriteExec {
+ def writingTask: WritingSparkTask[W]
+
+ protected override def writeWithV2(batchWrite: BatchWrite): Seq[InternalRow] = {
+ val rdd: RDD[InternalRow] = {
+ val tempRdd = query.execute()
+ // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single
+ // partition rdd to make sure we at least set up one write task to write the metadata.
+ if (tempRdd.partitions.length == 0) {
+ sparkContext.parallelize(Seq.empty[InternalRow], 1)
+ } else {
+ tempRdd
+ }
+ }
+ // introduce a local var to avoid serializing the whole class
+ val task = writingTask
+ val writerFactory = batchWrite.createBatchWriterFactory(
+ PhysicalWriteInfoImpl(rdd.getNumPartitions))
+ val useCommitCoordinator = batchWrite.useCommitCoordinator
+ val messages = new Array[WriterCommitMessage](rdd.partitions.length)
+ val totalNumRowsAccumulator = new LongAccumulator()
+
+ logInfo(s"Start processing data source write support: $batchWrite. " +
+ s"The input RDD has ${messages.length} partitions.")
+
+ // Avoid object not serializable issue.
+ val writeMetrics: Map[String, SQLMetric] = customMetrics
+
+ try {
+ sparkContext.runJob(
+ rdd,
+ (context: TaskContext, iter: Iterator[InternalRow]) =>
+ task.run(writerFactory, context, iter, useCommitCoordinator, writeMetrics),
+ rdd.partitions.indices,
+ (index, result: DataWritingSparkTaskResult) => {
+ val commitMessage = result.writerCommitMessage
+ messages(index) = commitMessage
+ totalNumRowsAccumulator.add(result.numRows)
+ batchWrite.onDataWriterCommit(commitMessage)
+ }
+ )
+
+ logInfo(s"Data source write support $batchWrite is committing.")
+ batchWrite.commit(messages)
+ logInfo(s"Data source write support $batchWrite committed.")
+ commitProgress = Some(StreamWriterCommitProgress(totalNumRowsAccumulator.value))
+ } catch {
+ case cause: Throwable =>
+ logError(s"Data source write support $batchWrite is aborting.")
+ try {
+ batchWrite.abort(messages)
+ } catch {
+ case t: Throwable =>
+ logError(s"Data source write support $batchWrite failed to abort.")
+ cause.addSuppressed(t)
+ throw QueryExecutionErrors.writingJobFailedError(cause)
+ }
+ logError(s"Data source write support $batchWrite aborted.")
+ cause match {
+ // Only wrap non fatal exceptions.
+ case NonFatal(e) => throw QueryExecutionErrors.writingJobAbortedError(e)
+ case _ => throw cause
+ }
+ }
+
+ Nil
+ }
+}
+
+trait WritingSparkTask[W <: DataWriter[InternalRow]] extends Logging with Serializable {
+
+ protected def writeFunc(writer: W, row: InternalRow): Unit
+
+ def run(
+ writerFactory: DataWriterFactory,
+ context: TaskContext,
+ iter: Iterator[InternalRow],
+ useCommitCoordinator: Boolean,
+ customMetrics: Map[String, SQLMetric]): DataWritingSparkTaskResult = {
+ val stageId = context.stageId()
+ val stageAttempt = context.stageAttemptNumber()
+ val partId = context.partitionId()
+ val taskId = context.taskAttemptId()
+ val attemptId = context.attemptNumber()
+ val dataWriter = writerFactory.createWriter(partId, taskId).asInstanceOf[W]
+
+ var count = 0L
+ // write the data and commit this writer.
+ Utils.tryWithSafeFinallyAndFailureCallbacks(block = {
+ while (iter.hasNext) {
+ if (count % CustomMetrics.NUM_ROWS_PER_UPDATE == 0) {
+ CustomMetrics.updateMetrics(ArraySeq.unsafeWrapArray(dataWriter.currentMetricsValues), customMetrics)
+ }
+
+ // Count is here.
+ count += 1
+ writeFunc(dataWriter, iter.next())
+ }
+
+ CustomMetrics.updateMetrics(ArraySeq.unsafeWrapArray(dataWriter.currentMetricsValues), customMetrics)
+
+ val msg = if (useCommitCoordinator) {
+ val coordinator = SparkEnv.get.outputCommitCoordinator
+ val commitAuthorized = coordinator.canCommit(stageId, stageAttempt, partId, attemptId)
+ if (commitAuthorized) {
+ logInfo(s"Commit authorized for partition $partId (task $taskId, attempt $attemptId, " +
+ s"stage $stageId.$stageAttempt)")
+ dataWriter.commit()
+ } else {
+ val commitDeniedException = QueryExecutionErrors.commitDeniedError(
+ partId, taskId, attemptId, stageId, stageAttempt)
+ logInfo(commitDeniedException.getMessage)
+ // throwing CommitDeniedException will trigger the catch block for abort
+ throw commitDeniedException
+ }
+
+ } else {
+ logInfo(s"Writer for partition ${context.partitionId()} is committing.")
+ dataWriter.commit()
+ }
+
+ logInfo(s"Committed partition $partId (task $taskId, attempt $attemptId, " +
+ s"stage $stageId.$stageAttempt)")
+
+ DataWritingSparkTaskResult(count, msg)
+
+ })(catchBlock = {
+ // If there is an error, abort this writer
+ logError(s"Aborting commit for partition $partId (task $taskId, attempt $attemptId, " +
+ s"stage $stageId.$stageAttempt)")
+ dataWriter.abort()
+ logError(s"Aborted commit for partition $partId (task $taskId, attempt $attemptId, " +
+ s"stage $stageId.$stageAttempt)")
+ }, finallyBlock = {
+ dataWriter.close()
+ })
+ }
+}
+
+case class DeltaWithMetadataWritingSparkTask(
+ projs: WriteDeltaProjections) extends WritingSparkTask[DeltaWriter[InternalRow]] {
+
+ private lazy val rowProjection = projs.rowProjection.orNull
+ private lazy val rowIdProjection = projs.rowIdProjection
+ private lazy val metadataProjection = projs.metadataProjection.orNull
+
+ override protected def writeFunc(writer: DeltaWriter[InternalRow], row: InternalRow): Unit = {
+ val operation = row.getInt(0)
+
+ operation match {
+ case DELETE_OPERATION =>
+ rowIdProjection.project(row)
+ metadataProjection.project(row)
+ writer.delete(metadataProjection, rowIdProjection)
+
+ case UPDATE_OPERATION =>
+ rowProjection.project(row)
+ rowIdProjection.project(row)
+ metadataProjection.project(row)
+ writer.update(metadataProjection, rowIdProjection, rowProjection)
+
+ case INSERT_OPERATION =>
+ rowProjection.project(row)
+ writer.insert(rowProjection)
+
+ case other =>
+ throw new SparkException(s"Unexpected operation ID: $other")
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
new file mode 100644
index 000000000..0685c88df
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.dynamicpruning
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.expressions.And
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.AttributeMap
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.DynamicPruningSubquery
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.PredicateHelper
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.planning.RewrittenRowLevelCommand
+import org.apache.spark.sql.catalyst.plans.LeftSemi
+import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.plans.logical.Join
+import org.apache.spark.sql.catalyst.plans.logical.JoinHint
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.plans.logical.RowLevelCommand
+import org.apache.spark.sql.catalyst.plans.logical.Sort
+import org.apache.spark.sql.catalyst.plans.logical.Subquery
+import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION
+import org.apache.spark.sql.catalyst.trees.TreePattern.SORT
+import org.apache.spark.sql.connector.read.SupportsRuntimeFiltering
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
+import scala.collection.compat.immutable.ArraySeq
+
+/**
+ * A rule that adds a runtime filter for row-level commands.
+ *
+ * Note that only group-based rewrite plans (i.e. ReplaceData) are taken into account.
+ * Row-based rewrite plans are subject to usual runtime filtering.
+ */
+case class RowLevelCommandDynamicPruning(spark: SparkSession) extends Rule[LogicalPlan] with PredicateHelper {
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
+ // apply special dynamic filtering only for plans that don't support deltas
+ case RewrittenRowLevelCommand(
+ command: RowLevelCommand,
+ DataSourceV2ScanRelation(_, scan: SupportsRuntimeFiltering, _),
+ rewritePlan: ReplaceData) if conf.dynamicPartitionPruningEnabled && isCandidate(command) =>
+
+ // use reference equality to find exactly the required scan relations
+ val newRewritePlan = rewritePlan transformUp {
+ case r: DataSourceV2ScanRelation if r.scan eq scan =>
+ val pruningKeys = ExtendedV2ExpressionUtils.resolveRefs[Attribute](
+ ArraySeq.unsafeWrapArray(scan.filterAttributes), r)
+ val dynamicPruningCond = buildDynamicPruningCondition(r, command, pruningKeys)
+ val filter = Filter(dynamicPruningCond, r)
+ // always optimize dynamic filtering subqueries for row-level commands as it is important
+ // to rewrite introduced predicates as joins because Spark recently stopped optimizing
+ // dynamic subqueries to facilitate broadcast reuse
+ optimizeSubquery(filter)
+ }
+ command.withNewRewritePlan(newRewritePlan)
+ }
+
+ private def isCandidate(command: RowLevelCommand): Boolean = command.condition match {
+ case Some(cond) if cond != Literal.TrueLiteral => true
+ case _ => false
+ }
+
+ private def buildDynamicPruningCondition(
+ relation: DataSourceV2ScanRelation,
+ command: RowLevelCommand,
+ pruningKeys: Seq[Attribute]): Expression = {
+
+ // construct a filtering plan with the original scan relation
+ val matchingRowsPlan = command match {
+ case d: DeleteFromIcebergTable =>
+ Filter(d.condition.get, relation)
+
+ case u: UpdateIcebergTable =>
+ // UPDATEs with subqueries are rewritten using a UNION with two identical scan relations
+ // the analyzer clones of them and assigns fresh expr IDs so that attributes don't collide
+ // this rule assigns dynamic filters to both scan relations based on the update condition
+ // the condition always refers to the original expr IDs and must be transformed
+ // see RewriteUpdateTable for more details
+ val attrMap = buildAttrMap(u.table.output, relation.output)
+ val transformedCond = u.condition.get transform {
+ case attr: AttributeReference if attrMap.contains(attr) => attrMap(attr)
+ }
+ Filter(transformedCond, relation)
+
+ case m: MergeIntoIcebergTable =>
+ Join(relation, m.sourceTable, LeftSemi, Some(m.mergeCondition), JoinHint.NONE)
+ }
+
+ // clone the original relation in the filtering plan and assign new expr IDs to avoid conflicts
+ val transformedMatchingRowsPlan = matchingRowsPlan transformUpWithNewOutput {
+ case r: DataSourceV2ScanRelation if r eq relation =>
+ val oldOutput = r.output
+ val newOutput = oldOutput.map(_.newInstance())
+ r.copy(output = newOutput) -> oldOutput.zip(newOutput)
+ }
+
+ val filterableScan = relation.scan.asInstanceOf[SupportsRuntimeFiltering]
+ val buildKeys = ExtendedV2ExpressionUtils.resolveRefs[Attribute](
+ ArraySeq.unsafeWrapArray(filterableScan.filterAttributes),
+ transformedMatchingRowsPlan)
+ val buildQuery = Project(buildKeys, transformedMatchingRowsPlan)
+ val dynamicPruningSubqueries = pruningKeys.zipWithIndex.map { case (key, index) =>
+ DynamicPruningSubquery(key, buildQuery, buildKeys, index, onlyInBroadcast = false)
+ }
+
+ // combine all dynamic subqueries to produce the final condition
+ dynamicPruningSubqueries.reduce(And)
+ }
+
+ private def buildAttrMap(
+ tableAttrs: Seq[Attribute],
+ scanAttrs: Seq[Attribute]): AttributeMap[Attribute] = {
+
+ val resolver = conf.resolver
+ val attrMapping = tableAttrs.flatMap { tableAttr =>
+ scanAttrs
+ .find(scanAttr => resolver(scanAttr.name, tableAttr.name))
+ .map(scanAttr => tableAttr -> scanAttr)
+ }
+ AttributeMap(attrMapping)
+ }
+
+ // borrowed from OptimizeSubqueries in Spark
+ private def optimizeSubquery(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+ _.containsPattern(PLAN_EXPRESSION)) {
+ case s: SubqueryExpression =>
+ val Subquery(newPlan, _) = spark.sessionState.optimizer.execute(Subquery.fromExpression(s))
+ // At this point we have an optimized subquery plan that we are going to attach
+ // to this subquery expression. Here we can safely remove any top level sort
+ // in the plan as tuples produced by a subquery are un-ordered.
+ s.withNewPlan(removeTopLevelSort(newPlan))
+ }
+
+ // borrowed from OptimizeSubqueries in Spark
+ private def removeTopLevelSort(plan: LogicalPlan): LogicalPlan = {
+ if (!plan.containsPattern(SORT)) {
+ return plan
+ }
+ plan match {
+ case Sort(_, _, child) => child
+ case Project(fields, child) => Project(fields, removeTopLevelSort(child))
+ case other => other
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
new file mode 100644
index 000000000..51ac57855
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.Objects;
+
+public class Employee {
+ private Integer id;
+ private String dep;
+
+ public Employee() {
+ }
+
+ public Employee(Integer id, String dep) {
+ this.id = id;
+ this.dep = dep;
+ }
+
+ public Integer getId() {
+ return id;
+ }
+
+ public void setId(Integer id) {
+ this.id = id;
+ }
+
+ public String getDep() {
+ return dep;
+ }
+
+ public void setDep(String dep) {
+ this.dep = dep;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ Employee employee = (Employee) other;
+ return Objects.equals(id, employee.id) && Objects.equals(dep, employee.dep);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, dep);
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java
new file mode 100644
index 000000000..32b45b820
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.hive.HiveCatalog;
+import org.apache.iceberg.hive.TestHiveMetastore;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.spark.SparkCatalogTestBase;
+import org.apache.iceberg.spark.SparkTestBase;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.internal.SQLConf;
+import org.junit.BeforeClass;
+
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS;
+
+public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase {
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ public SparkExtensionsTestBase(String catalogName, String implementation, Map<String, String> config) {
+ super(catalogName, implementation, config);
+ }
+
+ @BeforeClass
+ public static void startMetastoreAndSpark() {
+ SparkTestBase.metastore = new TestHiveMetastore();
+ metastore.start();
+ SparkTestBase.hiveConf = metastore.hiveConf();
+
+ SparkTestBase.spark = SparkSession.builder()
+ .master("local[2]")
+ .config("spark.testing", "true")
+ .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic")
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname))
+ .config("spark.sql.shuffle.partitions", "4")
+ .config("spark.sql.hive.metastorePartitionPruningFallbackOnException", "true")
+ .config(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean()))
+ .enableHiveSupport()
+ .getOrCreate();
+
+ SparkTestBase.catalog = (HiveCatalog)
+ CatalogUtil.loadCatalog(HiveCatalog.class.getName(), "hive", ImmutableMap.of(), hiveConf);
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
new file mode 100644
index 000000000..278559a56
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.spark.SparkCatalog;
+import org.apache.iceberg.spark.SparkSessionCatalog;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoder;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.junit.Assert;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import static org.apache.iceberg.DataOperations.DELETE;
+import static org.apache.iceberg.DataOperations.OVERWRITE;
+import static org.apache.iceberg.SnapshotSummary.ADDED_DELETE_FILES_PROP;
+import static org.apache.iceberg.SnapshotSummary.ADDED_FILES_PROP;
+import static org.apache.iceberg.SnapshotSummary.CHANGED_PARTITION_COUNT_PROP;
+import static org.apache.iceberg.SnapshotSummary.DELETED_FILES_PROP;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE;
+
+@RunWith(Parameterized.class)
+public abstract class SparkRowLevelOperationsTestBase extends SparkExtensionsTestBase {
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ protected final String fileFormat;
+ protected final boolean vectorized;
+ protected final String distributionMode;
+
+ public SparkRowLevelOperationsTestBase(String catalogName, String implementation,
+ Map<String, String> config, String fileFormat,
+ boolean vectorized,
+ String distributionMode) {
+ super(catalogName, implementation, config);
+ this.fileFormat = fileFormat;
+ this.vectorized = vectorized;
+ this.distributionMode = distributionMode;
+ }
+
+ @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}," +
+ " format = {3}, vectorized = {4}, distributionMode = {5}")
+ public static Object[][] parameters() {
+ return new Object[][] {
+ { "testhive", SparkCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default"
+ ),
+ "orc",
+ true,
+ WRITE_DISTRIBUTION_MODE_NONE
+ },
+ { "testhive", SparkCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default"
+ ),
+ "parquet",
+ true,
+ WRITE_DISTRIBUTION_MODE_NONE
+ },
+ { "testhadoop", SparkCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hadoop"
+ ),
+ "parquet",
+ RANDOM.nextBoolean(),
+ WRITE_DISTRIBUTION_MODE_HASH
+ },
+ { "spark_catalog", SparkSessionCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default",
+ "clients", "1",
+ "parquet-enabled", "false",
+ "cache-enabled", "false" // Spark will delete tables using v1, leaving the cache out of sync
+ ),
+ "avro",
+ false,
+ WRITE_DISTRIBUTION_MODE_RANGE
+ }
+ };
+ }
+
+ protected abstract Map<String, String> extraTableProperties();
+
+ protected void initTable() {
+ sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')", tableName, DEFAULT_FILE_FORMAT, fileFormat);
+ sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')", tableName, WRITE_DISTRIBUTION_MODE, distributionMode);
+
+ switch (fileFormat) {
+ case "parquet":
+ sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%b')", tableName, PARQUET_VECTORIZATION_ENABLED, vectorized);
+ break;
+ case "orc":
+ Assert.assertTrue(vectorized);
+ break;
+ case "avro":
+ Assert.assertFalse(vectorized);
+ break;
+ }
+
+ Map<String, String> props = extraTableProperties();
+ props.forEach((prop, value) -> {
+ sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')", tableName, prop, value);
+ });
+ }
+
+ protected void createAndInitTable(String schema) {
+ createAndInitTable(schema, null);
+ }
+
+ protected void createAndInitTable(String schema, String jsonData) {
+ sql("CREATE TABLE %s (%s) USING iceberg", tableName, schema);
+ initTable();
+
+ if (jsonData != null) {
+ try {
+ Dataset<Row> ds = toDS(schema, jsonData);
+ ds.writeTo(tableName).append();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException("Failed to write data", e);
+ }
+ }
+ }
+
+ protected void append(String table, String jsonData) {
+ append(table, null, jsonData);
+ }
+
+ protected void append(String table, String schema, String jsonData) {
+ try {
+ Dataset<Row> ds = toDS(schema, jsonData);
+ ds.coalesce(1).writeTo(table).append();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException("Failed to write data", e);
+ }
+ }
+
+ protected void createOrReplaceView(String name, String jsonData) {
+ createOrReplaceView(name, null, jsonData);
+ }
+
+ protected void createOrReplaceView(String name, String schema, String jsonData) {
+ Dataset<Row> ds = toDS(schema, jsonData);
+ ds.createOrReplaceTempView(name);
+ }
+
+ protected <T> void createOrReplaceView(String name, List<T> data, Encoder<T> encoder) {
+ spark.createDataset(data, encoder).createOrReplaceTempView(name);
+ }
+
+ private Dataset<Row> toDS(String schema, String jsonData) {
+ List<String> jsonRows = Arrays.stream(jsonData.split("\n"))
+ .filter(str -> str.trim().length() > 0)
+ .collect(Collectors.toList());
+ Dataset<String> jsonDS = spark.createDataset(jsonRows, Encoders.STRING());
+
+ if (schema != null) {
+ return spark.read().schema(schema).json(jsonDS);
+ } else {
+ return spark.read().json(jsonDS);
+ }
+ }
+
+ protected void validateDelete(Snapshot snapshot, String changedPartitionCount, String deletedDataFiles) {
+ validateSnapshot(snapshot, DELETE, changedPartitionCount, deletedDataFiles, null, null);
+ }
+
+ protected void validateCopyOnWrite(Snapshot snapshot, String changedPartitionCount,
+ String deletedDataFiles, String addedDataFiles) {
+ validateSnapshot(snapshot, OVERWRITE, changedPartitionCount, deletedDataFiles, null, addedDataFiles);
+ }
+
+ protected void validateMergeOnRead(Snapshot snapshot, String changedPartitionCount,
+ String addedDeleteFiles, String addedDataFiles) {
+ validateSnapshot(snapshot, OVERWRITE, changedPartitionCount, null, addedDeleteFiles, addedDataFiles);
+ }
+
+ protected void validateSnapshot(Snapshot snapshot, String operation, String changedPartitionCount,
+ String deletedDataFiles, String addedDeleteFiles, String addedDataFiles) {
+ Assert.assertEquals("Operation must match", operation, snapshot.operation());
+ validateProperty(snapshot, CHANGED_PARTITION_COUNT_PROP, changedPartitionCount);
+ validateProperty(snapshot, DELETED_FILES_PROP, deletedDataFiles);
+ validateProperty(snapshot, ADDED_DELETE_FILES_PROP, addedDeleteFiles);
+ validateProperty(snapshot, ADDED_FILES_PROP, addedDataFiles);
+ }
+
+ protected void validateProperty(Snapshot snapshot, String property, Set<String> expectedValues) {
+ String actual = snapshot.summary().get(property);
+ Assert.assertTrue("Snapshot property " + property + " has unexpected value, actual = " +
+ actual + ", expected one of : " + String.join(",", expectedValues),
+ expectedValues.contains(actual));
+ }
+
+ protected void validateProperty(Snapshot snapshot, String property, String expectedValue) {
+ String actual = snapshot.summary().get(property);
+ Assert.assertEquals("Snapshot property " + property + " has unexpected value.", expectedValue, actual);
+ }
+
+ protected void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
new file mode 100644
index 000000000..e274ad857
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
@@ -0,0 +1,903 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumWriter;
+import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.joda.time.DateTime;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class TestAddFilesProcedure extends SparkExtensionsTestBase {
+
+ private final String sourceTableName = "source_table";
+ private File fileTableDir;
+
+ public TestAddFilesProcedure(String catalogName, String implementation, Map<String, String> config) {
+ super(catalogName, implementation, config);
+ }
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ @Before
+ public void setupTempDirs() {
+ try {
+ fileTableDir = temp.newFolder();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @After
+ public void dropTables() {
+ sql("DROP TABLE IF EXISTS %s", sourceTableName);
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @Test
+ public void addDataUnpartitioned() {
+ createUnpartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Ignore // TODO Classpath issues prevent us from actually writing to a Spark ORC table
+ public void addDataUnpartitionedOrc() {
+ createUnpartitionedFileTable("orc");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`orc`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addAvroFile() throws Exception {
+ // Spark Session Catalog cannot load metadata tables
+ // with "The namespace in session catalog must have exactly one name part"
+ Assume.assumeFalse(catalogName.equals("spark_catalog"));
+
+ // Create an Avro file
+
+ Schema schema = SchemaBuilder.record("record").fields()
+ .requiredInt("id")
+ .requiredString("data")
+ .endRecord();
+ GenericRecord record1 = new GenericData.Record(schema);
+ record1.put("id", 1L);
+ record1.put("data", "a");
+ GenericRecord record2 = new GenericData.Record(schema);
+ record2.put("id", 2L);
+ record2.put("data", "b");
+ File outputFile = temp.newFile("test.avro");
+
+ DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter(schema);
+ DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter(datumWriter);
+ dataFileWriter.create(schema, outputFile);
+ dataFileWriter.append(record1);
+ dataFileWriter.append(record2);
+ dataFileWriter.close();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Long, data String) USING iceberg";
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`avro`.`%s`')",
+ catalogName, tableName, outputFile.getPath());
+ Assert.assertEquals(1L, result);
+
+ List<Object[]> expected = Lists.newArrayList(
+ new Object[]{1L, "a"},
+ new Object[]{2L, "b"}
+ );
+
+ assertEquals("Iceberg table contains correct data",
+ expected,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+
+ List<Object[]> actualRecordCount = sql("select %s from %s.files",
+ DataFile.RECORD_COUNT.name(),
+ tableName);
+ List<Object[]> expectedRecordCount = Lists.newArrayList();
+ expectedRecordCount.add(new Object[]{2L});
+ assertEquals("Iceberg file metadata should have correct metadata count",
+ expectedRecordCount, actualRecordCount);
+ }
+
+ // TODO Adding spark-avro doesn't work in tests
+ @Ignore
+ public void addDataUnpartitionedAvro() {
+ createUnpartitionedFileTable("avro");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`avro`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addDataUnpartitionedHive() {
+ createUnpartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '%s')",
+ catalogName, tableName, sourceTableName);
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addDataUnpartitionedExtraCol() {
+ createUnpartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String, foo string) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addDataUnpartitionedMissingCol() {
+ createUnpartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addDataPartitionedMissingCol() {
+ createPartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(8L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addDataPartitioned() {
+ createPartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(8L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Ignore // TODO Classpath issues prevent us from actually writing to a Spark ORC table
+ public void addDataPartitionedOrc() {
+ createPartitionedFileTable("orc");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(8L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ // TODO Adding spark-avro doesn't work in tests
+ @Ignore
+ public void addDataPartitionedAvro() {
+ createPartitionedFileTable("avro");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`avro`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(8L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addDataPartitionedHive() {
+ createPartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '%s')",
+ catalogName, tableName, sourceTableName);
+
+ Assert.assertEquals(8L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addPartitionToPartitioned() {
+ createPartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addDataPartitionedByDateToPartitioned() {
+ createDatePartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, date Date) USING iceberg PARTITIONED BY (date)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('date', '2021-01-01'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, date FROM %s WHERE date = '2021-01-01' ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, date FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addFilteredPartitionsToPartitioned() {
+ createCompositePartitionedTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg " +
+ "PARTITIONED BY (id, dept)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addFilteredPartitionsToPartitioned2() {
+ createCompositePartitionedTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg " +
+ "PARTITIONED BY (id, dept)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(6L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnId() {
+ createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg " +
+ "PARTITIONED BY (id, dept)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnDept() {
+ createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg " +
+ "PARTITIONED BY (id, dept)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ Assert.assertEquals(6L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addWeirdCaseHiveTable() {
+ createWeirdCaseTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, `naMe` String, dept String, subdept String) USING iceberg " +
+ "PARTITIONED BY (`naMe`)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '%s', map('naMe', 'John Doe'))",
+ catalogName, tableName, sourceTableName);
+
+ Assert.assertEquals(2L, result);
+
+ /*
+ While we would like to use
+ SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id
+ Spark does not actually handle this pushdown correctly for hive based tables and it returns 0 records
+ */
+ List<Object[]> expected =
+ sql("SELECT id, `naMe`, dept, subdept from %s ORDER BY id", sourceTableName)
+ .stream()
+ .filter(r -> r[1].equals("John Doe"))
+ .collect(Collectors.toList());
+
+ // TODO when this assert breaks Spark fixed the pushdown issue
+ Assert.assertEquals("If this assert breaks it means that Spark has fixed the pushdown issue", 0,
+ sql("SELECT id, `naMe`, dept, subdept from %s WHERE `naMe` = 'John Doe' ORDER BY id", sourceTableName)
+ .size());
+
+ // Pushdown works for iceberg
+ Assert.assertEquals("We should be able to pushdown mixed case partition keys", 2,
+ sql("SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id", tableName)
+ .size());
+
+ assertEquals("Iceberg table contains correct data",
+ expected,
+ sql("SELECT id, `naMe`, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void addPartitionToPartitionedHive() {
+ createPartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result = scalarSql("CALL %s.system.add_files('%s', '%s', map('id', 1))",
+ catalogName, tableName, sourceTableName);
+
+ Assert.assertEquals(2L, result);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void invalidDataImport() {
+ createPartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ AssertHelpers.assertThrows("Should forbid adding of partitioned data to unpartitioned table",
+ IllegalArgumentException.class,
+ "Cannot use partition filter with an unpartitioned table",
+ () -> scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath())
+ );
+
+ AssertHelpers.assertThrows("Should forbid adding of partitioned data to unpartitioned table",
+ IllegalArgumentException.class,
+ "Cannot add partitioned files to an unpartitioned table",
+ () -> scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath())
+ );
+ }
+
+ @Test
+ public void invalidDataImportPartitioned() {
+ createUnpartitionedFileTable("parquet");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ AssertHelpers.assertThrows("Should forbid adding with a mismatching partition spec",
+ IllegalArgumentException.class,
+ "is greater than the number of partitioned columns",
+ () -> scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('x', '1', 'y', '2'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath()));
+
+ AssertHelpers.assertThrows("Should forbid adding with partition spec with incorrect columns",
+ IllegalArgumentException.class,
+ "specified partition filter refers to columns that are not partitioned",
+ () -> scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', '2'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath()));
+ }
+
+
+ @Test
+ public void addTwice() {
+ createPartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result1 = scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s', " +
+ "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName);
+ Assert.assertEquals(2L, result1);
+
+ Object result2 = scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s', " +
+ "partition_filter => map('id', 2))",
+ catalogName, tableName, sourceTableName);
+ Assert.assertEquals(2L, result2);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", tableName));
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", tableName));
+ }
+
+ @Test
+ public void duplicateDataPartitioned() {
+ createPartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s', " +
+ "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName);
+
+ AssertHelpers.assertThrows("Should not allow adding duplicate files",
+ IllegalStateException.class,
+ "Cannot complete import because data files to be imported already" +
+ " exist within the target table",
+ () -> scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s', " +
+ "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName));
+ }
+
+ @Test
+ public void duplicateDataPartitionedAllowed() {
+ createPartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result1 = scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s', " +
+ "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName);
+
+ Assert.assertEquals(2L, result1);
+
+ Object result2 = scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s', " +
+ "partition_filter => map('id', 1)," +
+ "check_duplicate_files => false)",
+ catalogName, tableName, sourceTableName);
+
+ Assert.assertEquals(2L, result2);
+
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 UNION ALL " +
+ "SELECT id, name, dept, subdept FROM %s WHERE id = 1", sourceTableName, sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s", tableName, tableName));
+ }
+
+ @Test
+ public void duplicateDataUnpartitioned() {
+ createUnpartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ scalarSql("CALL %s.system.add_files('%s', '%s')",
+ catalogName, tableName, sourceTableName);
+
+ AssertHelpers.assertThrows("Should not allow adding duplicate files",
+ IllegalStateException.class,
+ "Cannot complete import because data files to be imported already" +
+ " exist within the target table",
+ () -> scalarSql("CALL %s.system.add_files('%s', '%s')",
+ catalogName, tableName, sourceTableName));
+ }
+
+ @Test
+ public void duplicateDataUnpartitionedAllowed() {
+ createUnpartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result1 = scalarSql("CALL %s.system.add_files('%s', '%s')",
+ catalogName, tableName, sourceTableName);
+ Assert.assertEquals(2L, result1);
+
+ Object result2 = scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s'," +
+ "check_duplicate_files => false)",
+ catalogName, tableName, sourceTableName);
+ Assert.assertEquals(2L, result2);
+
+ assertEquals("Iceberg table contains correct data",
+ sql("SELECT * FROM (SELECT * FROM %s UNION ALL " +
+ "SELECT * from %s) ORDER BY id", sourceTableName, sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+
+
+ }
+
+ @Test
+ public void testEmptyImportDoesNotThrow() {
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+ sql(createIceberg, tableName);
+
+ // Empty path based import
+ Object pathResult = scalarSql("CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+ Assert.assertEquals(0L, pathResult);
+ assertEquals("Iceberg table contains no added data when importing from an empty path",
+ emptyQueryResult,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+
+ // Empty table based import
+ String createHive = "CREATE TABLE %s (id Integer, name String, dept String, subdept String) STORED AS parquet";
+ sql(createHive, sourceTableName);
+
+ Object tableResult = scalarSql("CALL %s.system.add_files('%s', '%s')",
+ catalogName, tableName, sourceTableName);
+ Assert.assertEquals(0L, tableResult);
+ assertEquals("Iceberg table contains no added data when importing from an empty table",
+ emptyQueryResult,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Test
+ public void testPartitionedImportFromEmptyPartitionDoesNotThrow() {
+ createPartitionedHiveTable();
+
+ final int emptyPartitionId = 999;
+ // Add an empty partition to the hive table
+ sql("ALTER TABLE %s ADD PARTITION (id = '%d') LOCATION '%d'", sourceTableName,
+ emptyPartitionId, emptyPartitionId);
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object tableResult = scalarSql("CALL %s.system.add_files(" +
+ "table => '%s', " +
+ "source_table => '%s', " +
+ "partition_filter => map('id', %d))",
+ catalogName, tableName, sourceTableName, emptyPartitionId);
+
+ Assert.assertEquals(0L, tableResult);
+ assertEquals("Iceberg table contains no added data when importing from an empty table",
+ emptyQueryResult,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ private static final List<Object[]> emptyQueryResult = Lists.newArrayList();
+
+ private static final StructField[] struct = {
+ new StructField("id", DataTypes.IntegerType, true, Metadata.empty()),
+ new StructField("name", DataTypes.StringType, true, Metadata.empty()),
+ new StructField("dept", DataTypes.StringType, true, Metadata.empty()),
+ new StructField("subdept", DataTypes.StringType, true, Metadata.empty())
+ };
+
+ private static final Dataset<Row> unpartitionedDF =
+ spark.createDataFrame(
+ ImmutableList.of(
+ RowFactory.create(1, "John Doe", "hr", "communications"),
+ RowFactory.create(2, "Jane Doe", "hr", "salary"),
+ RowFactory.create(3, "Matt Doe", "hr", "communications"),
+ RowFactory.create(4, "Will Doe", "facilities", "all")),
+ new StructType(struct)).repartition(1);
+
+ private static final Dataset<Row> singleNullRecordDF =
+ spark.createDataFrame(
+ ImmutableList.of(
+ RowFactory.create(null, null, null, null)),
+ new StructType(struct)).repartition(1);
+
+ private static final Dataset<Row> partitionedDF =
+ unpartitionedDF.select("name", "dept", "subdept", "id");
+
+ private static final Dataset<Row> compositePartitionedDF =
+ unpartitionedDF.select("name", "subdept", "id", "dept");
+
+ private static final Dataset<Row> compositePartitionedNullRecordDF =
+ singleNullRecordDF.select("name", "subdept", "id", "dept");
+
+ private static final Dataset<Row> weirdColumnNamesDF =
+ unpartitionedDF.select(
+ unpartitionedDF.col("id"),
+ unpartitionedDF.col("subdept"),
+ unpartitionedDF.col("dept"),
+ unpartitionedDF.col("name").as("naMe"));
+
+ private static final StructField[] dateStruct = {
+ new StructField("id", DataTypes.IntegerType, true, Metadata.empty()),
+ new StructField("name", DataTypes.StringType, true, Metadata.empty()),
+ new StructField("dept", DataTypes.StringType, true, Metadata.empty()),
+ new StructField("ts", DataTypes.DateType, true, Metadata.empty())
+ };
+
+ private static java.sql.Date toDate(String value) {
+ return new java.sql.Date(DateTime.parse(value).getMillis());
+ }
+
+ private static final Dataset<Row> dateDF =
+ spark.createDataFrame(
+ ImmutableList.of(
+ RowFactory.create(1, "John Doe", "hr", toDate("2021-01-01")),
+ RowFactory.create(2, "Jane Doe", "hr", toDate("2021-01-01")),
+ RowFactory.create(3, "Matt Doe", "hr", toDate("2021-01-02")),
+ RowFactory.create(4, "Will Doe", "facilities", toDate("2021-01-02"))),
+ new StructType(dateStruct)).repartition(2);
+
+ private void createUnpartitionedFileTable(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s LOCATION '%s'";
+
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+ unpartitionedDF.write().insertInto(sourceTableName);
+ unpartitionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createPartitionedFileTable(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s PARTITIONED BY (id) " +
+ "LOCATION '%s'";
+
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ partitionedDF.write().insertInto(sourceTableName);
+ partitionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createCompositePartitionedTable(String format) {
+ String createParquet = "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s " +
+ "PARTITIONED BY (id, dept) LOCATION '%s'";
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ compositePartitionedDF.write().insertInto(sourceTableName);
+ compositePartitionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createCompositePartitionedTableWithNullValueInPartitionColumn(String format) {
+ String createParquet = "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s " +
+ "PARTITIONED BY (id, dept) LOCATION '%s'";
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ Dataset<Row> unionedDF = compositePartitionedDF.unionAll(compositePartitionedNullRecordDF)
+ .select("name", "subdept", "id", "dept")
+ .repartition(1);
+
+ unionedDF.write().insertInto(sourceTableName);
+ unionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createWeirdCaseTable() {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, subdept String, dept String) " +
+ "PARTITIONED BY (`naMe` String) STORED AS parquet";
+
+ sql(createParquet, sourceTableName);
+
+ weirdColumnNamesDF.write().insertInto(sourceTableName);
+ weirdColumnNamesDF.write().insertInto(sourceTableName);
+
+ }
+
+ private void createUnpartitionedHiveTable() {
+ String createHive = "CREATE TABLE %s (id Integer, name String, dept String, subdept String) STORED AS parquet";
+
+ sql(createHive, sourceTableName);
+
+ unpartitionedDF.write().insertInto(sourceTableName);
+ unpartitionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createPartitionedHiveTable() {
+ String createHive = "CREATE TABLE %s (name String, dept String, subdept String) " +
+ "PARTITIONED BY (id Integer) STORED AS parquet";
+
+ sql(createHive, sourceTableName);
+
+ partitionedDF.write().insertInto(sourceTableName);
+ partitionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createDatePartitionedFileTable(String format) {
+ String createParquet = "CREATE TABLE %s (id Integer, name String, dept String, date Date) USING %s " +
+ "PARTITIONED BY (date) LOCATION '%s'";
+
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ dateDF.write().insertInto(sourceTableName);
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
new file mode 100644
index 000000000..9d630508b
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -0,0 +1,415 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.Map;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.spark.source.SparkTable;
+import org.apache.spark.sql.connector.catalog.CatalogManager;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAlterTablePartitionFields extends SparkExtensionsTestBase {
+ public TestAlterTablePartitionFields(String catalogName, String implementation, Map<String, String> config) {
+ super(catalogName, implementation, config);
+ }
+
+ @After
+ public void removeTable() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @Test
+ public void testAddIdentityPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD category", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .identity("category")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testAddBucketPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .bucket("id", 16, "id_bucket_16")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testAddTruncatePartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD truncate(data, 4)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .truncate("data", 4, "data_trunc_4")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testAddYearsPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .year("ts")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testAddMonthsPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD months(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .month("ts")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testAddDaysPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .day("ts")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testAddHoursPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hours(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .hour("ts")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testAddNamedPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .bucket("id", 16, "shard")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testDropIdentityPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, data string) USING iceberg PARTITIONED BY (category)",
+ tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertEquals("Table should start with 1 partition field", 1, table.spec().fields().size());
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD category", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .alwaysNull("category", "category")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testDropDaysPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, ts timestamp, data string) USING iceberg PARTITIONED BY (days(ts))",
+ tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertEquals("Table should start with 1 partition field", 1, table.spec().fields().size());
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD days(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .alwaysNull("ts", "ts_day")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testDropBucketPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg PARTITIONED BY (bucket(16, id))",
+ tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertEquals("Table should start with 1 partition field", 1, table.spec().fields().size());
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD bucket(16, id)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .alwaysNull("id", "id_bucket")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testDropPartitionByName() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
+
+ table.refresh();
+
+ Assert.assertEquals("Table should have 1 partition field", 1, table.spec().fields().size());
+
+ // Should be recognized as iceberg command even with extra white spaces
+ sql("ALTER TABLE %s DROP PARTITION \n FIELD shard", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("id", "shard")
+ .build();
+
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+ }
+
+ @Test
+ public void testReplacePartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
+ table.refresh();
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .day("ts")
+ .build();
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD days(ts) WITH hours(ts)", tableName);
+ table.refresh();
+ expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "ts_day")
+ .hour("ts")
+ .build();
+ Assert.assertEquals("Should changed from daily to hourly partitioned field", expected, table.spec());
+ }
+
+ @Test
+ public void testReplacePartitionAndRename() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
+ table.refresh();
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .day("ts")
+ .build();
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD days(ts) WITH hours(ts) AS hour_col", tableName);
+ table.refresh();
+ expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "ts_day")
+ .hour("ts", "hour_col")
+ .build();
+ Assert.assertEquals("Should changed from daily to hourly partitioned field", expected, table.spec());
+ }
+
+ @Test
+ public void testReplaceNamedPartition() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts) AS day_col", tableName);
+ table.refresh();
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .day("ts", "day_col")
+ .build();
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD day_col WITH hours(ts)", tableName);
+ table.refresh();
+ expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "day_col")
+ .hour("ts")
+ .build();
+ Assert.assertEquals("Should changed from daily to hourly partitioned field", expected, table.spec());
+ }
+
+ @Test
+ public void testReplaceNamedPartitionAndRenameDifferently() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned());
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts) AS day_col", tableName);
+ table.refresh();
+ PartitionSpec expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .day("ts", "day_col")
+ .build();
+ Assert.assertEquals("Should have new spec field", expected, table.spec());
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD day_col WITH hours(ts) AS hour_col", tableName);
+ table.refresh();
+ expected = PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "day_col")
+ .hour("ts", "hour_col")
+ .build();
+ Assert.assertEquals("Should changed from daily to hourly partitioned field", expected, table.spec());
+ }
+
+ @Test
+ public void testSparkTableAddDropPartitions() throws Exception {
+ sql("CREATE TABLE %s (id bigint NOT NULL, ts timestamp, data string) USING iceberg", tableName);
+ Assert.assertEquals("spark table partition should be empty", 0, sparkTable().partitioning().length);
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
+ assertPartitioningEquals(sparkTable(), 1, "bucket(16, id)");
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD truncate(data, 4)", tableName);
+ assertPartitioningEquals(sparkTable(), 2, "truncate(data, 4)");
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName);
+ assertPartitioningEquals(sparkTable(), 3, "years(ts)");
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD years(ts)", tableName);
+ assertPartitioningEquals(sparkTable(), 2, "truncate(data, 4)");
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD truncate(data, 4)", tableName);
+ assertPartitioningEquals(sparkTable(), 1, "bucket(16, id)");
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD shard", tableName);
+ sql("DESCRIBE %s", tableName);
+ Assert.assertEquals("spark table partition should be empty", 0, sparkTable().partitioning().length);
+ }
+
+ private void assertPartitioningEquals(SparkTable table, int len, String transform) {
+ Assert.assertEquals("spark table partition should be " + len, len, table.partitioning().length);
+ Assert.assertEquals("latest spark table partition transform should match",
+ transform, table.partitioning()[len - 1].toString());
+ }
+
+ private SparkTable sparkTable() throws Exception {
+ validationCatalog.loadTable(tableIdent).refresh();
+ CatalogManager catalogManager = spark.sessionState().catalogManager();
+ TableCatalog catalog = (TableCatalog) catalogManager.catalog(catalogName);
+ Identifier identifier = Identifier.of(tableIdent.namespace().levels(), tableIdent.name());
+ return (SparkTable) catalog.loadTable(identifier);
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
new file mode 100644
index 000000000..ac12953d0
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.Map;
+import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAlterTableSchema extends SparkExtensionsTestBase {
+ public TestAlterTableSchema(String catalogName, String implementation, Map<String, String> config) {
+ super(catalogName, implementation, config);
+ }
+
+ @After
+ public void removeTable() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @Test
+ public void testSetIdentifierFields() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, " +
+ "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start without identifier", table.schema().identifierFieldIds().isEmpty());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id", tableName);
+ table.refresh();
+ Assert.assertEquals("Should have new identifier field",
+ Sets.newHashSet(table.schema().findField("id").fieldId()),
+ table.schema().identifierFieldIds());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ Assert.assertEquals("Should have new identifier field",
+ Sets.newHashSet(
+ table.schema().findField("id").fieldId(),
+ table.schema().findField("location.lon").fieldId()),
+ table.schema().identifierFieldIds());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS location.lon", tableName);
+ table.refresh();
+ Assert.assertEquals("Should have new identifier field",
+ Sets.newHashSet(table.schema().findField("location.lon").fieldId()),
+ table.schema().identifierFieldIds());
+ }
+
+ @Test
+ public void testSetInvalidIdentifierFields() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, id2 bigint) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start without identifier", table.schema().identifierFieldIds().isEmpty());
+ AssertHelpers.assertThrows("should not allow setting unknown fields",
+ IllegalArgumentException.class,
+ "not found in current schema or added columns",
+ () -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS unknown", tableName));
+
+ AssertHelpers.assertThrows("should not allow setting optional fields",
+ IllegalArgumentException.class,
+ "not a required field",
+ () -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS id2", tableName));
+ }
+
+ @Test
+ public void testDropIdentifierFields() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, " +
+ "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start without identifier", table.schema().identifierFieldIds().isEmpty());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ Assert.assertEquals("Should have new identifier fields",
+ Sets.newHashSet(
+ table.schema().findField("id").fieldId(),
+ table.schema().findField("location.lon").fieldId()),
+ table.schema().identifierFieldIds());
+
+ sql("ALTER TABLE %s DROP IDENTIFIER FIELDS id", tableName);
+ table.refresh();
+ Assert.assertEquals("Should removed identifier field",
+ Sets.newHashSet(table.schema().findField("location.lon").fieldId()),
+ table.schema().identifierFieldIds());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ Assert.assertEquals("Should have new identifier fields",
+ Sets.newHashSet(
+ table.schema().findField("id").fieldId(),
+ table.schema().findField("location.lon").fieldId()),
+ table.schema().identifierFieldIds());
+
+ sql("ALTER TABLE %s DROP IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ Assert.assertEquals("Should have no identifier field",
+ Sets.newHashSet(),
+ table.schema().identifierFieldIds());
+ }
+
+ @Test
+ public void testDropInvalidIdentifierFields() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string NOT NULL, " +
+ "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ Assert.assertTrue("Table should start without identifier", table.schema().identifierFieldIds().isEmpty());
+ AssertHelpers.assertThrows("should not allow dropping unknown fields",
+ IllegalArgumentException.class,
+ "field unknown not found",
+ () -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS unknown", tableName));
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id", tableName);
+ AssertHelpers.assertThrows("should not allow dropping a field that is not an identifier",
+ IllegalArgumentException.class,
+ "data is not an identifier field",
+ () -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS data", tableName));
+
+ AssertHelpers.assertThrows("should not allow dropping a nested field that is not an identifier",
+ IllegalArgumentException.class,
+ "location.lon is not an identifier field",
+ () -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS location.lon", tableName));
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java
new file mode 100644
index 000000000..baf464d94
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.spark.sql.AnalysisException;
+import org.junit.After;
+import org.junit.Test;
+
+public class TestAncestorsOfProcedure extends SparkExtensionsTestBase {
+
+ public TestAncestorsOfProcedure(String catalogName, String implementation, Map<String, String> config) {
+ super(catalogName, implementation, config);
+ }
+
+ @After
+ public void removeTables() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @Test
+ public void testAncestorOfUsingEmptyArgs() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'b')", tableName);
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Long currentSnapshotId = table.currentSnapshot().snapshotId();
+ Long currentTimestamp = table.currentSnapshot().timestampMillis();
+ Long preSnapshotId = table.currentSnapshot().parentId();
+ Long preTimeStamp = table.snapshot(table.currentSnapshot().parentId()).timestampMillis();
+
+ List<Object[]> output = sql("CALL %s.system.ancestors_of('%s')",
+ catalogName, tableIdent);
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(currentSnapshotId, currentTimestamp),
+ row(preSnapshotId, preTimeStamp)),
+ output);
+ }
+
+ @Test
+ public void testAncestorOfUsingSnapshotId() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'b')", tableName);
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Long currentSnapshotId = table.currentSnapshot().snapshotId();
+ Long currentTimestamp = table.currentSnapshot().timestampMillis();
+ Long preSnapshotId = table.currentSnapshot().parentId();
+ Long preTimeStamp = table.snapshot(table.currentSnapshot().parentId()).timestampMillis();
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(currentSnapshotId, currentTimestamp),
+ row(preSnapshotId, preTimeStamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, currentSnapshotId));
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(row(preSnapshotId, preTimeStamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, preSnapshotId));
+ }
+
+ @Test
+ public void testAncestorOfWithRollBack() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+ table.refresh();
+ Long firstSnapshotId = table.currentSnapshot().snapshotId();
+ Long firstTimestamp = table.currentSnapshot().timestampMillis();
+ sql("INSERT INTO TABLE %s VALUES (2, 'b')", tableName);
+ table.refresh();
+ Long secondSnapshotId = table.currentSnapshot().snapshotId();
+ Long secondTimestamp = table.currentSnapshot().timestampMillis();
+ sql("INSERT INTO TABLE %s VALUES (3, 'c')", tableName);
+ table.refresh();
+ Long thirdSnapshotId = table.currentSnapshot().snapshotId();
+ Long thirdTimestamp = table.currentSnapshot().timestampMillis();
+
+ // roll back
+ sql("CALL %s.system.rollback_to_snapshot('%s', %dL)",
+ catalogName, tableIdent, secondSnapshotId);
+
+ sql("INSERT INTO TABLE %s VALUES (4, 'd')", tableName);
+ table.refresh();
+ Long fourthSnapshotId = table.currentSnapshot().snapshotId();
+ Long fourthTimestamp = table.currentSnapshot().timestampMillis();
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(fourthSnapshotId, fourthTimestamp),
+ row(secondSnapshotId, secondTimestamp),
+ row(firstSnapshotId, firstTimestamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, fourthSnapshotId));
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(thirdSnapshotId, thirdTimestamp),
+ row(secondSnapshotId, secondTimestamp),
+ row(firstSnapshotId, firstTimestamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, thirdSnapshotId));
+ }
+
+ @Test
+ public void testAncestorOfUsingNamedArgs() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Long firstSnapshotId = table.currentSnapshot().snapshotId();
+ Long firstTimestamp = table.currentSnapshot().timestampMillis();
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(row(firstSnapshotId, firstTimestamp)),
+ sql("CALL %s.system.ancestors_of(snapshot_id => %dL, table => '%s')",
+ catalogName, firstSnapshotId, tableIdent));
+ }
+
+ @Test
+ public void testInvalidAncestorOfCases() {
+ AssertHelpers.assertThrows("Should reject calls without all required args",
+ AnalysisException.class, "Missing required parameters",
+ () -> sql("CALL %s.system.ancestors_of()", catalogName));
+
+ AssertHelpers.assertThrows("Should reject calls with empty table identifier",
+ IllegalArgumentException.class, "Cannot handle an empty identifier for argument table",
+ () -> sql("CALL %s.system.ancestors_of('')", catalogName));
+
+ AssertHelpers.assertThrows("Should reject calls with invalid arg types",
+ AnalysisException.class, "Wrong arg type for snapshot_id: cannot cast",
+ () -> sql("CALL %s.system.ancestors_of('%s', 1.1)", catalogName, tableIdent));
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java
new file mode 100644
index 000000000..a6815d21a
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.List;
+import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.expressions.Expression;
+import org.apache.spark.sql.catalyst.expressions.Literal;
+import org.apache.spark.sql.catalyst.expressions.Literal$;
+import org.apache.spark.sql.catalyst.parser.ParseException;
+import org.apache.spark.sql.catalyst.parser.ParserInterface;
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergParseException;
+import org.apache.spark.sql.catalyst.plans.logical.CallArgument;
+import org.apache.spark.sql.catalyst.plans.logical.CallStatement;
+import org.apache.spark.sql.catalyst.plans.logical.NamedArgument;
+import org.apache.spark.sql.catalyst.plans.logical.PositionalArgument;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import scala.collection.JavaConverters;
+
+public class TestCallStatementParser {
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ private static SparkSession spark = null;
+ private static ParserInterface parser = null;
+
+ @BeforeClass
+ public static void startSpark() {
+ TestCallStatementParser.spark = SparkSession.builder()
+ .master("local[2]")
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.extra.prop", "value")
+ .getOrCreate();
+ TestCallStatementParser.parser = spark.sessionState().sqlParser();
+ }
+
+ @AfterClass
+ public static void stopSpark() {
+ SparkSession currentSpark = TestCallStatementParser.spark;
+ TestCallStatementParser.spark = null;
+ TestCallStatementParser.parser = null;
+ currentSpark.stop();
+ }
+
+ @Test
+ public void testCallWithPositionalArgs() throws ParseException {
+ CallStatement call = (CallStatement) parser.parsePlan("CALL c.n.func(1, '2', 3L, true, 1.0D, 9.0e1, 900e-1BD)");
+ Assert.assertEquals(ImmutableList.of("c", "n", "func"), JavaConverters.seqAsJavaList(call.name()));
+
+ Assert.assertEquals(7, call.args().size());
+
+ checkArg(call, 0, 1, DataTypes.IntegerType);
+ checkArg(call, 1, "2", DataTypes.StringType);
+ checkArg(call, 2, 3L, DataTypes.LongType);
+ checkArg(call, 3, true, DataTypes.BooleanType);
+ checkArg(call, 4, 1.0D, DataTypes.DoubleType);
+ checkArg(call, 5, 9.0e1, DataTypes.DoubleType);
+ checkArg(call, 6, new BigDecimal("900e-1"), DataTypes.createDecimalType(3, 1));
+ }
+
+ @Test
+ public void testCallWithNamedArgs() throws ParseException {
+ CallStatement call = (CallStatement) parser.parsePlan("CALL cat.system.func(c1 => 1, c2 => '2', c3 => true)");
+ Assert.assertEquals(ImmutableList.of("cat", "system", "func"), JavaConverters.seqAsJavaList(call.name()));
+
+ Assert.assertEquals(3, call.args().size());
+
+ checkArg(call, 0, "c1", 1, DataTypes.IntegerType);
+ checkArg(call, 1, "c2", "2", DataTypes.StringType);
+ checkArg(call, 2, "c3", true, DataTypes.BooleanType);
+ }
+
+ @Test
+ public void testCallWithMixedArgs() throws ParseException {
+ CallStatement call = (CallStatement) parser.parsePlan("CALL cat.system.func(c1 => 1, '2')");
+ Assert.assertEquals(ImmutableList.of("cat", "system", "func"), JavaConverters.seqAsJavaList(call.name()));
+
+ Assert.assertEquals(2, call.args().size());
+
+ checkArg(call, 0, "c1", 1, DataTypes.IntegerType);
+ checkArg(call, 1, "2", DataTypes.StringType);
+ }
+
+ @Test
+ public void testCallWithTimestampArg() throws ParseException {
+ CallStatement call = (CallStatement) parser.parsePlan("CALL cat.system.func(TIMESTAMP '2017-02-03T10:37:30.00Z')");
+ Assert.assertEquals(ImmutableList.of("cat", "system", "func"), JavaConverters.seqAsJavaList(call.name()));
+
+ Assert.assertEquals(1, call.args().size());
+
+ checkArg(call, 0, Timestamp.from(Instant.parse("2017-02-03T10:37:30.00Z")), DataTypes.TimestampType);
+ }
+
+ @Test
+ public void testCallWithVarSubstitution() throws ParseException {
+ CallStatement call = (CallStatement) parser.parsePlan("CALL cat.system.func('${spark.extra.prop}')");
+ Assert.assertEquals(ImmutableList.of("cat", "system", "func"), JavaConverters.seqAsJavaList(call.name()));
+
+ Assert.assertEquals(1, call.args().size());
+
+ checkArg(call, 0, "value", DataTypes.StringType);
+ }
+
+ @Test
+ public void testCallParseError() {
+ AssertHelpers.assertThrows("Should fail with a sensible parse error", IcebergParseException.class,
+ "missing '(' at 'radish'",
+ () -> parser.parsePlan("CALL cat.system radish kebab"));
+ }
+
+ @Test
+ public void testCallStripsComments() throws ParseException {
+ List<String> callStatementsWithComments = Lists.newArrayList(
+ "/* bracketed comment */ CALL cat.system.func('${spark.extra.prop}')",
+ "/**/ CALL cat.system.func('${spark.extra.prop}')",
+ "-- single line comment \n CALL cat.system.func('${spark.extra.prop}')",
+ "-- multiple \n-- single line \n-- comments \n CALL cat.system.func('${spark.extra.prop}')",
+ "/* select * from multiline_comment \n where x like '%sql%'; */ CALL cat.system.func('${spark.extra.prop}')",
+ "/* {\"app\": \"dbt\", \"dbt_version\": \"1.0.1\", \"profile_name\": \"profile1\", \"target_name\": \"dev\", " +
+ "\"node_id\": \"model.profile1.stg_users\"} \n*/ CALL cat.system.func('${spark.extra.prop}')",
+ "/* Some multi-line comment \n" +
+ "*/ CALL /* inline comment */ cat.system.func('${spark.extra.prop}') -- ending comment",
+ "CALL -- a line ending comment\n" +
+ "cat.system.func('${spark.extra.prop}')"
+ );
+ for (String sqlText : callStatementsWithComments) {
+ CallStatement call = (CallStatement) parser.parsePlan(sqlText);
+ Assert.assertEquals(ImmutableList.of("cat", "system", "func"), JavaConverters.seqAsJavaList(call.name()));
+
+ Assert.assertEquals(1, call.args().size());
+
+ checkArg(call, 0, "value", DataTypes.StringType);
+ }
+ }
+
+ private void checkArg(CallStatement call, int index, Object expectedValue, DataType expectedType) {
+ checkArg(call, index, null, expectedValue, expectedType);
+ }
+
+ private void checkArg(CallStatement call, int index, String expectedName,
+ Object expectedValue, DataType expectedType) {
+
+ if (expectedName != null) {
+ NamedArgument arg = checkCast(call.args().apply(index), NamedArgument.class);
+ Assert.assertEquals(expectedName, arg.name());
+ } else {
+ CallArgument arg = call.args().apply(index);
+ checkCast(arg, PositionalArgument.class);
+ }
+
+ Expression expectedExpr = toSparkLiteral(expectedValue, expectedType);
+ Expression actualExpr = call.args().apply(index).expr();
+ Assert.assertEquals("Arg types must match", expectedExpr.dataType(), actualExpr.dataType());
+ Assert.assertEquals("Arg must match", expectedExpr, actualExpr);
+ }
+
+ private Literal toSparkLiteral(Object value, DataType dataType) {
+ return Literal$.MODULE$.create(value, dataType);
+ }
+
+ private <T> T checkCast(Object value, Class<T> expectedClass) {
+ Assert.assertTrue("Expected instance of " + expectedClass.getName(), expectedClass.isInstance(value));
+ return expectedClass.cast(value);
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java
new file mode 100644
index 000000000..c69964693
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.spark.sql.AnalysisException;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.catalyst.analysis.NoSuchProcedureException;
+import org.junit.After;
+import org.junit.Test;
+
+import static org.apache.iceberg.TableProperties.WRITE_AUDIT_PUBLISH_ENABLED;
+
+public class TestCherrypickSnapshotProcedure extends SparkExtensionsTestBase {
+
+ public TestCherrypickSnapshotProcedure(String catalogName, String implementation, Map<String, String> config) {
+ super(catalogName, implementation, config);
+ }
+
+ @After
+ public void removeTables() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @Test
+ public void testCherrypickSnapshotUsingPositionalArgs() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')", tableName, WRITE_AUDIT_PUBLISH_ENABLED);
+
+ spark.conf().set("spark.wap.id", "1");
+
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+
+ assertEquals("Should not see rows from staged snapshot",
+ ImmutableList.of(),
+ sql("SELECT * FROM %s", tableName));
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Snapshot wapSnapshot = Iterables.getOnlyElement(table.snapshots());
+
+ List<Object[]> output = sql(
+ "CALL %s.system.cherrypick_snapshot('%s', %dL)",
+ catalogName, tableIdent, wapSnapshot.snapshotId());
+
+ table.refresh();
+
+ Snapshot currentSnapshot = table.currentSnapshot();
+
+ assertEquals("Procedure output must match",
+ ImmutableList.of(row(wapSnapshot.snapshotId(), currentSnapshot.snapshotId())),
+ output);
+
+ assertEquals("Cherrypick must be successful",
+ ImmutableList.of(row(1L, "a")),
+ sql("SELECT * FROM %s", tableName));
+ }
+
+ @Test
+ public void testCherrypickSnapshotUsingNamedArgs() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')", tableName, WRITE_AUDIT_PUBLISH_ENABLED);
+
+ spark.conf().set("spark.wap.id", "1");
+
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+
+ assertEquals("Should not see rows from staged snapshot",
+ ImmutableList.of(),
+ sql("SELECT * FROM %s", tableName));
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Snapshot wapSnapshot = Iterables.getOnlyElement(table.snapshots());
+
+ List<Object[]> output = sql(
+ "CALL %s.system.cherrypick_snapshot(snapshot_id => %dL, table => '%s')",
+ catalogName, wapSnapshot.snapshotId(), tableIdent);
+
+ table.refresh();
+
+ Snapshot currentSnapshot = table.currentSnapshot();
+
+ assertEquals("Procedure output must match",
+ ImmutableList.of(row(wapSnapshot.snapshotId(), currentSnapshot.snapshotId())),
+ output);
+
+ assertEquals("Cherrypick must be successful",
+ ImmutableList.of(row(1L, "a")),
+ sql("SELECT * FROM %s", tableName));
+ }
+
+ @Test
+ public void testCherrypickSnapshotRefreshesRelationCache() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')", tableName, WRITE_AUDIT_PUBLISH_ENABLED);
+
+ Dataset<Row> query = spark.sql("SELECT * FROM " + tableName + " WHERE id = 1");
+ query.createOrReplaceTempView("tmp");
+
+ spark.sql("CACHE TABLE tmp");
+
+ assertEquals("View should not produce rows", ImmutableList.of(), sql("SELECT * FROM tmp"));
+
+ spark.conf().set("spark.wap.id", "1");
+
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+
+ assertEquals("Should not see rows from staged snapshot",
+ ImmutableList.of(),
+ sql("SELECT * FROM %s", tableName));
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Snapshot wapSnapshot = Iterables.getOnlyElement(table.snapshots());
+
+ sql("CALL %s.system.cherrypick_snapshot('%s', %dL)",
+ catalogName, tableIdent, wapSnapshot.snapshotId());
+
+ assertEquals("Cherrypick snapshot should be visible",
+ ImmutableList.of(row(1L, "a")),
+ sql("SELECT * FROM tmp"));
+
+ sql("UNCACHE TABLE tmp");
+ }
+
+ @Test
+ public void testCherrypickInvalidSnapshot() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+
+ AssertHelpers.assertThrows("Should reject invalid snapshot id",
+ ValidationException.class, "Cannot cherry-pick unknown snapshot ID",
+ () -> sql("CALL %s.system.cherrypick_snapshot('%s', -1L)", catalogName, tableIdent));
+ }
+
+ @Test
+ public void testInvalidCherrypickSnapshotCases() {
+ AssertHelpers.assertThrows("Should not allow mixed args",
+ AnalysisException.class, "Named and positional arguments cannot be mixed",
+ () -> sql("CALL %s.system.cherrypick_snapshot('n', table => 't', 1L)", catalogName));
+
+ AssertHelpers.assertThrows("Should not resolve procedures in arbitrary namespaces",
+ NoSuchProcedureException.class, "not found",
+ () -> sql("CALL %s.custom.cherrypick_snapshot('n', 't', 1L)", catalogName));
+
+ AssertHelpers.assertThrows("Should reject calls without all required args",
+ AnalysisException.class, "Missing required parameters",
+ () -> sql("CALL %s.system.cherrypick_snapshot('t')", catalogName));
+
+ AssertHelpers.assertThrows("Should reject calls with empty table identifier",
+ IllegalArgumentException.class, "Cannot handle an empty identifier",
+ () -> sql("CALL %s.system.cherrypick_snapshot('', 1L)", catalogName));
+
+ AssertHelpers.assertThrows("Should reject calls with invalid arg types",
+ AnalysisException.class, "Wrong arg type for snapshot_id: cannot cast",
+ () -> sql("CALL %s.system.cherrypick_snapshot('t', 2.2)", catalogName));
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestConflictValidation.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestConflictValidation.java
new file mode 100644
index 000000000..4ce44818a
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestConflictValidation.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.extensions;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.IsolationLevel;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.SparkWriteOptions;
+import org.apache.iceberg.spark.source.SimpleRecord;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.functions;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestConflictValidation extends SparkExtensionsTestBase {
+
+ public TestConflictValidation(String catalogName, String implementation, Map<String, String> config) {
+ super(catalogName, implementation, config);
+ }
+
+ @Before
+ public void createTables() {
+ sql("CREATE TABLE %s (id int, data string) USING iceberg " +
+ "PARTITIONED BY (id)" +
+ "TBLPROPERTIES" +
+ "('format-version'='2'," +
+ "'write.delete.mode'='merge-on-read')", tableName);
+ sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName);
+ }
+
+ @After
+ public void removeTables() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @Test
+ public void testOverwriteFilterSerializableIsolation() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+ long snapshotId = table.currentSnapshot().snapshotId();
+
+ List<SimpleRecord> records = Lists.newArrayList(new SimpleRecord(1, "a"));
+ spark.createDataFrame(records, SimpleRecord.class).writeTo(tableName).append();
+
+ // Validating from previous snapshot finds conflicts
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting new data files should throw exception",
+ ValidationException.class,
+ "Found conflicting files that can contain records matching ref(name=\"id\") == 1:",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ }
+
+ @Test
+ public void testOverwriteFilterSerializableIsolation2() throws Exception {
+ List<SimpleRecord> records = Lists.newArrayList(
+ new SimpleRecord(1, "a"),
+ new SimpleRecord(1, "b"));
+ spark.createDataFrame(records, SimpleRecord.class).coalesce(1).writeTo(tableName).append();
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ long snapshotId = table.currentSnapshot().snapshotId();
+
+ // This should add a delete file
+ sql("DELETE FROM %s WHERE id='1' and data='b'", tableName);
+ table.refresh();
+
+ // Validating from previous snapshot finds conflicts
+ List<SimpleRecord> conflictingRecords = Lists.newArrayList(new SimpleRecord(1, "a"));
+ Dataset<Row> conflictingDf = spark.createDataFrame(conflictingRecords, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting new delete files should throw exception",
+ ValidationException.class,
+ "Found new conflicting delete files that can apply to records matching ref(name=\"id\") == 1:",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ }
+
+ @Test
+ public void testOverwriteFilterSerializableIsolation3() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+ long snapshotId = table.currentSnapshot().snapshotId();
+
+ // This should delete a data file
+ sql("DELETE FROM %s WHERE id='1'", tableName);
+ table.refresh();
+
+ // Validating from previous snapshot finds conflicts
+ List<SimpleRecord> conflictingRecords = Lists.newArrayList(new SimpleRecord(1, "a"));
+ Dataset<Row> conflictingDf = spark.createDataFrame(conflictingRecords, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting deleted data files should throw exception",
+ ValidationException.class,
+ "Found conflicting deleted files that can contain records matching ref(name=\"id\") == 1:",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ }
+
+ @Test
+ public void testOverwriteFilterNoSnapshotIdValidation() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ List<SimpleRecord> records = Lists.newArrayList(new SimpleRecord(1, "a"));
+ spark.createDataFrame(records, SimpleRecord.class).writeTo(tableName).append();
+
+ // Validating from no snapshot id defaults to beginning snapshot id and finds conflicts
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting new data files should throw exception",
+ ValidationException.class,
+ "Found conflicting files that can contain records matching ref(name=\"id\") == 1:",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ }
+
+ @Test
+ public void testOverwriteFilterSnapshotIsolation() throws Exception {
+ List<SimpleRecord> records = Lists.newArrayList(
+ new SimpleRecord(1, "a"),
+ new SimpleRecord(1, "b"));
+ spark.createDataFrame(records, SimpleRecord.class).coalesce(1).writeTo(tableName).append();
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ long snapshotId = table.currentSnapshot().snapshotId();
+
+ // This should add a delete file
+ sql("DELETE FROM %s WHERE id='1' and data='b'", tableName);
+ table.refresh();
+
+ // Validating from previous snapshot finds conflicts
+ List<SimpleRecord> conflictingRecords = Lists.newArrayList(new SimpleRecord(1, "a"));
+ Dataset<Row> conflictingDf = spark.createDataFrame(conflictingRecords, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting new delete files should throw exception",
+ ValidationException.class,
+ "Found new conflicting delete files that can apply to records matching ref(name=\"id\") == 1:",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ }
+
+ @Test
+ public void testOverwriteFilterSnapshotIsolation2() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+ long snapshotId = table.currentSnapshot().snapshotId();
+
+ List<SimpleRecord> records = Lists.newArrayList(
+ new SimpleRecord(1, "a"));
+ spark.createDataFrame(records, SimpleRecord.class).writeTo(tableName).append();
+
+ // Validation should not fail due to conflicting data file in snapshot isolation mode
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwrite(functions.col("id").equalTo(1));
+ }
+
+ @Test
+ public void testOverwritePartitionSerializableIsolation() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+ final long snapshotId = table.currentSnapshot().snapshotId();
+
+ List<SimpleRecord> records = Lists.newArrayList(new SimpleRecord(1, "a"));
+ spark.createDataFrame(records, SimpleRecord.class).writeTo(tableName).append();
+
+ // Validating from previous snapshot finds conflicts
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting deleted data files should throw exception",
+ ValidationException.class,
+ "Found conflicting files that can contain records matching partitions [id=1]",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwritePartitions();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwritePartitions();
+ }
+
+ @Test
+ public void testOverwritePartitionSnapshotIsolation() throws Exception {
+ List<SimpleRecord> records = Lists.newArrayList(
+ new SimpleRecord(1, "a"),
+ new SimpleRecord(1, "b"));
+ spark.createDataFrame(records, SimpleRecord.class).coalesce(1).writeTo(tableName).append();
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ final long snapshotId = table.currentSnapshot().snapshotId();
+
+ // This should generate a delete file
+ sql("DELETE FROM %s WHERE data='a'", tableName);
+
+ // Validating from previous snapshot finds conflicts
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting deleted data files should throw exception",
+ ValidationException.class,
+ "Found new conflicting delete files that can apply to records matching [id=1]",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwritePartitions();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwritePartitions();
+ }
+
+ @Test
+ public void testOverwritePartitionSnapshotIsolation2() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+ final long snapshotId = table.currentSnapshot().snapshotId();
+
+ // This should delete a data file
+ sql("DELETE FROM %s WHERE id='1'", tableName);
+
+ // Validating from previous snapshot finds conflicts
+ List<SimpleRecord> records = Lists.newArrayList(
+ new SimpleRecord(1, "a"));
+ spark.createDataFrame(records, SimpleRecord.class).coalesce(1).writeTo(tableName).append();
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+
+ AssertHelpers.assertThrowsCause("Conflicting deleted data files should throw exception",
+ ValidationException.class,
+ "Found conflicting deleted files that can apply to records matching [id=1]",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwritePartitions();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long newSnapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(newSnapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwritePartitions();
+ }
+
+ @Test
+ public void testOverwritePartitionSnapshotIsolation3() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+ final long snapshotId = table.currentSnapshot().snapshotId();
+
+ List<SimpleRecord> records = Lists.newArrayList(new SimpleRecord(1, "a"));
+ spark.createDataFrame(records, SimpleRecord.class).writeTo(tableName).append();
+
+ // Validation should not find conflicting data file in snapshot isolation mode
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SNAPSHOT.toString())
+ .overwritePartitions();
+ }
+
+ @Test
+ public void testOverwritePartitionNoSnapshotIdValidation() throws Exception {
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ List<SimpleRecord> records = Lists.newArrayList(
+ new SimpleRecord(1, "a"));
+ spark.createDataFrame(records, SimpleRecord.class).writeTo(tableName).append();
+
+ // Validating from null snapshot is equivalent to validating from beginning
+ Dataset<Row> conflictingDf = spark.createDataFrame(records, SimpleRecord.class);
+ AssertHelpers.assertThrowsCause("Conflicting deleted data files should throw exception",
+ ValidationException.class,
+ "Found conflicting files that can contain records matching partitions [id=1]",
+ () -> {
+ try {
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwritePartitions();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ // Validating from latest snapshot should succeed
+ table.refresh();
+ long snapshotId = table.currentSnapshot().snapshotId();
+ conflictingDf.writeTo(tableName)
+ .option(SparkWriteOptions.VALIDATE_FROM_SNAPSHOT_ID, String.valueOf(snapshotId))
+ .option(SparkWriteOptions.ISOLATION_LEVEL, IsolationLevel.SERIALIZABLE.toString())
+ .overwritePartitions();
+ }
+}
diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java
new file mode 100644
index 000000000..c9d159062
--- /dev/null
+++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
... 68622 lines suppressed ...