You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2025/02/20 09:42:12 UTC
(iceberg) branch main updated: Spark: Remove Spark 3.3 support (#12279)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new d6d97d1683 Spark: Remove Spark 3.3 support (#12279)
d6d97d1683 is described below
commit d6d97d1683465c08d3903673ca6ccbca2004f735
Author: Manu Zhang <Ow...@gmail.com>
AuthorDate: Thu Feb 20 17:42:05 2025 +0800
Spark: Remove Spark 3.3 support (#12279)
---
.github/workflows/publish-snapshot.yml | 2 +-
.github/workflows/spark-ci.yml | 4 +-
.gitignore | 2 -
build.gradle | 1 -
dev/stage-binaries.sh | 3 +-
docs/docs/aws.md | 18 +-
docs/docs/nessie.md | 8 +-
gradle.properties | 2 +-
gradle/libs.versions.toml | 1 -
jmh.gradle | 5 -
settings.gradle | 12 -
site/docs/multi-engine-support.md | 2 +-
spark/build.gradle | 4 -
spark/v3.3/build.gradle | 300 ---
.../spark/MergeCardinalityCheckBenchmark.java | 230 --
.../IcebergSqlExtensions.g4 | 374 ---
.../extensions/IcebergSparkSessionExtensions.scala | 77 -
.../spark/sql/catalyst/ProjectingInternalRow.scala | 123 -
.../analysis/AlignRowLevelCommandAssignments.scala | 113 -
.../AlignedRowLevelIcebergCommandCheck.scala | 38 -
.../analysis/AssignmentAlignmentSupport.scala | 202 --
.../analysis/CheckMergeIntoTableConditions.scala | 74 -
.../MergeIntoIcebergTableResolutionCheck.scala | 35 -
.../analysis/ProcedureArgumentCoercion.scala | 56 -
.../analysis/ResolveMergeIntoTableReferences.scala | 153 --
.../sql/catalyst/analysis/ResolveProcedures.scala | 190 --
.../analysis/RewriteDeleteFromIcebergTable.scala | 118 -
.../catalyst/analysis/RewriteMergeIntoTable.scala | 469 ----
.../analysis/RewriteRowLevelIcebergCommand.scala | 90 -
.../sql/catalyst/analysis/RewriteUpdateTable.scala | 183 --
.../sql/catalyst/expressions/AssignmentUtils.scala | 86 -
.../expressions/ExtendedV2ExpressionUtils.scala | 103 -
.../ExtendedReplaceNullWithFalseInPredicate.scala | 132 -
.../ExtendedSimplifyConditionalsInPredicate.scala | 94 -
.../IcebergSparkSqlExtensionsParser.scala | 409 ---
.../IcebergSqlExtensionsAstBuilder.scala | 385 ---
.../planning/RewrittenRowLevelCommand.scala | 113 -
.../catalyst/plans/logical/AddPartitionField.scala | 33 -
.../sql/catalyst/plans/logical/BranchOptions.scala | 23 -
.../spark/sql/catalyst/plans/logical/Call.scala | 33 -
.../plans/logical/CreateOrReplaceBranch.scala | 39 -
.../plans/logical/CreateOrReplaceTag.scala | 39 -
.../plans/logical/DeleteFromIcebergTable.scala | 49 -
.../sql/catalyst/plans/logical/DropBranch.scala | 33 -
.../plans/logical/DropIdentifierFields.scala | 34 -
.../plans/logical/DropPartitionField.scala | 33 -
.../spark/sql/catalyst/plans/logical/DropTag.scala | 33 -
.../plans/logical/MergeIntoIcebergTable.scala | 77 -
.../sql/catalyst/plans/logical/MergeRows.scala | 55 -
.../catalyst/plans/logical/NoStatsUnaryNode.scala | 31 -
.../plans/logical/ReplaceIcebergData.scala | 69 -
.../plans/logical/ReplacePartitionField.scala | 38 -
.../catalyst/plans/logical/RowLevelCommand.scala | 28 -
.../plans/logical/SetIdentifierFields.scala | 35 -
.../sql/catalyst/plans/logical/TagOptions.scala | 22 -
.../logical/UnresolvedMergeIntoIcebergTable.scala | 45 -
.../plans/logical/UpdateIcebergTable.scala | 53 -
.../plans/logical/V2WriteCommandLike.scala | 42 -
.../sql/catalyst/plans/logical/WriteDelta.scala | 126 -
.../sql/catalyst/plans/logical/statements.scala | 44 -
.../spark/sql/catalyst/util/RowDeltaUtils.scala | 27 -
.../sql/catalyst/util/WriteDeltaProjections.scala | 27 -
.../connector/expressions/TruncateTransform.scala | 38 -
.../write/ExtendedLogicalWriteInfoImpl.scala | 31 -
.../datasources/v2/AddPartitionFieldExec.scala | 56 -
.../sql/execution/datasources/v2/CallExec.scala | 40 -
.../datasources/v2/CreateOrReplaceBranchExec.scala | 100 -
.../datasources/v2/CreateOrReplaceTagExec.scala | 84 -
.../execution/datasources/v2/DropBranchExec.scala | 56 -
.../datasources/v2/DropIdentifierFieldsExec.scala | 65 -
.../datasources/v2/DropPartitionFieldExec.scala | 67 -
.../sql/execution/datasources/v2/DropTagExec.scala | 56 -
.../v2/ExtendedDataSourceV2Implicits.scala | 40 -
.../v2/ExtendedDataSourceV2Strategy.scala | 159 --
.../v2/ExtendedDistributionAndOrderingUtils.scala | 85 -
.../datasources/v2/ExtendedV2Writes.scala | 140 --
.../execution/datasources/v2/MergeRowsExec.scala | 180 --
...ptimizeMetadataOnlyDeleteFromIcebergTable.scala | 73 -
.../execution/datasources/v2/ReplaceDataExec.scala | 40 -
.../datasources/v2/ReplacePartitionFieldExec.scala | 72 -
.../v2/ReplaceRewrittenRowLevelCommand.scala | 34 -
.../v2/RowLevelCommandScanRelationPushDown.scala | 200 --
.../datasources/v2/SetIdentifierFieldsExec.scala | 53 -
.../v2/SetWriteDistributionAndOrderingExec.scala | 79 -
.../execution/datasources/v2/WriteDeltaExec.scala | 238 --
.../RowLevelCommandDynamicPruning.scala | 182 --
.../apache/iceberg/spark/extensions/Employee.java | 66 -
.../iceberg/spark/extensions/ProcedureUtil.java | 54 -
.../spark/extensions/SparkExtensionsTestBase.java | 71 -
.../SparkRowLevelOperationsTestBase.java | 350 ---
.../spark/extensions/TestAddFilesProcedure.java | 1161 ---------
.../extensions/TestAlterTablePartitionFields.java | 578 -----
.../spark/extensions/TestAlterTableSchema.java | 158 --
.../spark/extensions/TestAncestorsOfProcedure.java | 166 --
.../iceberg/spark/extensions/TestBranchDDL.java | 403 ---
.../spark/extensions/TestCallStatementParser.java | 250 --
.../spark/extensions/TestChangelogTable.java | 353 ---
.../TestCherrypickSnapshotProcedure.java | 205 --
.../spark/extensions/TestConflictValidation.java | 460 ----
.../spark/extensions/TestCopyOnWriteDelete.java | 189 --
.../spark/extensions/TestCopyOnWriteMerge.java | 202 --
.../spark/extensions/TestCopyOnWriteUpdate.java | 188 --
.../TestCreateChangelogViewProcedure.java | 639 -----
.../iceberg/spark/extensions/TestDelete.java | 1269 ----------
.../extensions/TestExpireSnapshotsProcedure.java | 578 -----
.../extensions/TestFastForwardBranchProcedure.java | 197 --
.../spark/extensions/TestIcebergExpressions.java | 105 -
.../apache/iceberg/spark/extensions/TestMerge.java | 2642 --------------------
.../spark/extensions/TestMergeOnReadDelete.java | 161 --
.../spark/extensions/TestMergeOnReadMerge.java | 47 -
.../spark/extensions/TestMergeOnReadUpdate.java | 47 -
.../TestMetaColumnProjectionWithStageScan.java | 128 -
.../spark/extensions/TestMetadataTables.java | 724 ------
.../extensions/TestMigrateTableProcedure.java | 224 --
.../extensions/TestPublishChangesProcedure.java | 197 --
.../extensions/TestRegisterTableProcedure.java | 84 -
.../extensions/TestRemoveOrphanFilesProcedure.java | 775 ------
.../spark/extensions/TestReplaceBranch.java | 277 --
.../TestRequiredDistributionAndOrdering.java | 319 ---
.../extensions/TestRewriteDataFilesProcedure.java | 840 -------
.../extensions/TestRewriteManifestsProcedure.java | 377 ---
.../extensions/TestRewritePositionDeleteFiles.java | 417 ---
.../TestRewritePositionDeleteFilesProcedure.java | 229 --
.../TestRollbackToSnapshotProcedure.java | 297 ---
.../TestRollbackToTimestampProcedure.java | 347 ---
.../TestSetCurrentSnapshotProcedure.java | 255 --
.../TestSetWriteDistributionAndOrdering.java | 328 ---
.../extensions/TestSnapshotTableProcedure.java | 230 --
...toragePartitionedJoinsInRowLevelOperations.java | 283 ---
.../iceberg/spark/extensions/TestTagDDL.java | 381 ---
.../iceberg/spark/extensions/TestUpdate.java | 1323 ----------
.../iceberg/spark/extensions/TestWriteAborts.java | 194 --
spark/v3.3/spark-runtime/LICENSE | 538 ----
spark/v3.3/spark-runtime/NOTICE | 403 ---
.../java/org/apache/iceberg/spark/SmokeTest.java | 175 --
.../apache/iceberg/spark/SparkBenchmarkUtil.java | 57 -
.../spark/action/DeleteOrphanFilesBenchmark.java | 180 --
.../action/IcebergSortCompactionBenchmark.java | 398 ---
.../iceberg/spark/action/RandomGeneratingUDF.java | 50 -
.../SparkParquetReadersFlatDataBenchmark.java | 227 --
.../SparkParquetReadersNestedDataBenchmark.java | 225 --
.../SparkParquetWritersFlatDataBenchmark.java | 130 -
.../SparkParquetWritersNestedDataBenchmark.java | 130 -
.../org/apache/iceberg/spark/source/Action.java | 24 -
.../spark/source/IcebergSourceBenchmark.java | 209 --
.../spark/source/IcebergSourceDeleteBenchmark.java | 340 ---
.../source/IcebergSourceFlatDataBenchmark.java | 59 -
.../source/IcebergSourceNestedDataBenchmark.java | 59 -
.../IcebergSourceNestedListDataBenchmark.java | 62 -
.../iceberg/spark/source/WritersBenchmark.java | 275 --
.../spark/source/avro/AvroWritersBenchmark.java | 39 -
.../IcebergSourceFlatAvroDataReadBenchmark.java | 142 --
.../IcebergSourceNestedAvroDataReadBenchmark.java | 142 --
.../orc/IcebergSourceFlatORCDataBenchmark.java | 68 -
.../orc/IcebergSourceFlatORCDataReadBenchmark.java | 210 --
...ebergSourceNestedListORCDataWriteBenchmark.java | 109 -
.../IcebergSourceNestedORCDataReadBenchmark.java | 183 --
...cebergSourceFlatParquetDataFilterBenchmark.java | 129 -
.../IcebergSourceFlatParquetDataReadBenchmark.java | 165 --
...IcebergSourceFlatParquetDataWriteBenchmark.java | 89 -
...gSourceNestedListParquetDataWriteBenchmark.java | 89 -
...bergSourceNestedParquetDataFilterBenchmark.java | 128 -
...cebergSourceNestedParquetDataReadBenchmark.java | 166 --
...ebergSourceNestedParquetDataWriteBenchmark.java | 88 -
.../IcebergSourceParquetEqDeleteBenchmark.java | 57 -
...ebergSourceParquetMultiDeleteFileBenchmark.java | 57 -
.../IcebergSourceParquetPosDeleteBenchmark.java | 60 -
...gSourceParquetWithUnrelatedDeleteBenchmark.java | 64 -
.../source/parquet/ParquetWritersBenchmark.java | 39 -
...dDictionaryEncodedFlatParquetDataBenchmark.java | 127 -
.../VectorizedReadFlatParquetDataBenchmark.java | 355 ---
.../VectorizedReadParquetDecimalBenchmark.java | 207 --
.../java/org/apache/iceberg/spark/BaseCatalog.java | 95 -
.../iceberg/spark/BaseFileRewriteCoordinator.java | 81 -
.../apache/iceberg/spark/ChangelogIterator.java | 127 -
.../org/apache/iceberg/spark/CommitMetadata.java | 66 -
.../iceberg/spark/ComputeUpdateIterator.java | 140 --
.../org/apache/iceberg/spark/ExtendedParser.java | 70 -
.../iceberg/spark/FileRewriteCoordinator.java | 32 -
.../org/apache/iceberg/spark/IcebergSpark.java | 56 -
.../org/apache/iceberg/spark/JobGroupInfo.java | 44 -
.../org/apache/iceberg/spark/JobGroupUtils.java | 46 -
.../org/apache/iceberg/spark/PathIdentifier.java | 57 -
.../spark/PositionDeletesRewriteCoordinator.java | 33 -
.../spark/PruneColumnsWithoutReordering.java | 240 --
.../iceberg/spark/RemoveCarryoverIterator.java | 148 --
.../iceberg/spark/RemoveNetCarryoverIterator.java | 129 -
.../apache/iceberg/spark/RollbackStagedTable.java | 142 --
.../apache/iceberg/spark/ScanTaskSetManager.java | 73 -
.../org/apache/iceberg/spark/SortOrderToSpark.java | 101 -
.../java/org/apache/iceberg/spark/Spark3Util.java | 1041 --------
.../org/apache/iceberg/spark/SparkAggregates.java | 84 -
.../iceberg/spark/SparkCachedTableCatalog.java | 211 --
.../org/apache/iceberg/spark/SparkCatalog.java | 812 ------
.../org/apache/iceberg/spark/SparkConfParser.java | 203 --
.../org/apache/iceberg/spark/SparkDataFile.java | 225 --
.../spark/SparkDistributionAndOrderingUtil.java | 239 --
.../apache/iceberg/spark/SparkExceptionUtil.java | 64 -
.../org/apache/iceberg/spark/SparkFilters.java | 277 --
.../iceberg/spark/SparkFixupTimestampType.java | 57 -
.../org/apache/iceberg/spark/SparkFixupTypes.java | 63 -
.../org/apache/iceberg/spark/SparkReadConf.java | 295 ---
.../org/apache/iceberg/spark/SparkReadOptions.java | 102 -
.../apache/iceberg/spark/SparkSQLProperties.java | 69 -
.../org/apache/iceberg/spark/SparkSchemaUtil.java | 393 ---
.../apache/iceberg/spark/SparkSessionCatalog.java | 398 ---
.../org/apache/iceberg/spark/SparkStructLike.java | 54 -
.../org/apache/iceberg/spark/SparkTableCache.java | 54 -
.../org/apache/iceberg/spark/SparkTableUtil.java | 995 --------
.../org/apache/iceberg/spark/SparkTypeToType.java | 158 --
.../org/apache/iceberg/spark/SparkTypeVisitor.java | 78 -
.../java/org/apache/iceberg/spark/SparkUtil.java | 351 ---
.../org/apache/iceberg/spark/SparkV2Filters.java | 334 ---
.../apache/iceberg/spark/SparkValueConverter.java | 159 --
.../org/apache/iceberg/spark/SparkWriteConf.java | 376 ---
.../apache/iceberg/spark/SparkWriteOptions.java | 82 -
.../org/apache/iceberg/spark/TypeToSparkType.java | 134 -
.../actions/BaseSnapshotUpdateSparkAction.java | 42 -
.../iceberg/spark/actions/BaseSparkAction.java | 446 ----
.../actions/BaseTableCreationSparkAction.java | 182 --
.../actions/DeleteOrphanFilesSparkAction.java | 676 -----
.../actions/DeleteReachableFilesSparkAction.java | 156 --
.../spark/actions/ExpireSnapshotsSparkAction.java | 270 --
.../org/apache/iceberg/spark/actions/FileInfo.java | 52 -
.../iceberg/spark/actions/ManifestFileBean.java | 156 --
.../spark/actions/MigrateTableSparkAction.java | 253 --
.../spark/actions/RewriteDataFilesSparkAction.java | 502 ----
.../spark/actions/RewriteManifestsSparkAction.java | 469 ----
.../RewritePositionDeleteFilesSparkAction.java | 465 ----
.../iceberg/spark/actions/SetAccumulator.java | 61 -
.../spark/actions/SnapshotTableSparkAction.java | 225 --
.../apache/iceberg/spark/actions/SparkActions.java | 99 -
.../spark/actions/SparkBinPackDataRewriter.java | 71 -
.../SparkBinPackPositionDeletesRewriter.java | 145 --
.../spark/actions/SparkShufflingDataRewriter.java | 154 --
.../spark/actions/SparkSizeBasedDataRewriter.java | 67 -
.../spark/actions/SparkSortDataRewriter.java | 65 -
.../spark/actions/SparkZOrderDataRewriter.java | 205 --
.../iceberg/spark/actions/SparkZOrderUDF.java | 325 ---
.../spark/data/AvroWithSparkSchemaVisitor.java | 76 -
.../spark/data/ParquetWithSparkSchemaVisitor.java | 231 --
.../apache/iceberg/spark/data/SparkAvroReader.java | 180 --
.../apache/iceberg/spark/data/SparkAvroWriter.java | 165 --
.../apache/iceberg/spark/data/SparkOrcReader.java | 135 -
.../iceberg/spark/data/SparkOrcValueReaders.java | 261 --
.../iceberg/spark/data/SparkOrcValueWriters.java | 221 --
.../apache/iceberg/spark/data/SparkOrcWriter.java | 236 --
.../iceberg/spark/data/SparkParquetReaders.java | 785 ------
.../iceberg/spark/data/SparkParquetWriters.java | 572 -----
.../iceberg/spark/data/SparkPlannedAvroReader.java | 192 --
.../iceberg/spark/data/SparkValueReaders.java | 322 ---
.../iceberg/spark/data/SparkValueWriters.java | 256 --
.../vectorized/ArrowVectorAccessorFactory.java | 132 -
.../data/vectorized/ArrowVectorAccessors.java | 38 -
.../spark/data/vectorized/ColumnVectorBuilder.java | 52 -
.../data/vectorized/ColumnVectorWithFilter.java | 95 -
.../spark/data/vectorized/ColumnarBatchReader.java | 255 --
.../data/vectorized/ConstantColumnVector.java | 122 -
.../spark/data/vectorized/DeletedColumnVector.java | 121 -
.../data/vectorized/IcebergArrowColumnVector.java | 158 --
.../data/vectorized/RowPositionColumnVector.java | 120 -
.../data/vectorized/VectorizedSparkOrcReaders.java | 497 ----
.../vectorized/VectorizedSparkParquetReaders.java | 135 -
.../iceberg/spark/functions/BucketFunction.java | 323 ---
.../iceberg/spark/functions/DaysFunction.java | 119 -
.../iceberg/spark/functions/HoursFunction.java | 90 -
.../spark/functions/IcebergVersionFunction.java | 96 -
.../iceberg/spark/functions/MonthsFunction.java | 118 -
.../iceberg/spark/functions/SparkFunctions.java | 57 -
.../iceberg/spark/functions/TruncateFunction.java | 357 ---
.../spark/functions/UnaryUnboundFunction.java | 44 -
.../iceberg/spark/functions/YearsFunction.java | 118 -
.../spark/procedures/AddFilesProcedure.java | 319 ---
.../spark/procedures/AncestorsOfProcedure.java | 115 -
.../iceberg/spark/procedures/BaseProcedure.java | 240 --
.../procedures/CherrypickSnapshotProcedure.java | 100 -
.../procedures/CreateChangelogViewProcedure.java | 283 ---
.../spark/procedures/ExpireSnapshotsProcedure.java | 173 --
.../procedures/FastForwardBranchProcedure.java | 94 -
.../spark/procedures/MigrateTableProcedure.java | 119 -
.../iceberg/spark/procedures/ProcedureInput.java | 241 --
.../spark/procedures/PublishChangesProcedure.java | 115 -
.../spark/procedures/RegisterTableProcedure.java | 110 -
.../procedures/RemoveOrphanFilesProcedure.java | 222 --
.../procedures/RewriteDataFilesProcedure.java | 228 --
.../procedures/RewriteManifestsProcedure.java | 121 -
.../RewritePositionDeleteFilesProcedure.java | 122 -
.../procedures/RollbackToSnapshotProcedure.java | 99 -
.../procedures/RollbackToTimestampProcedure.java | 104 -
.../procedures/SetCurrentSnapshotProcedure.java | 100 -
.../spark/procedures/SnapshotTableProcedure.java | 120 -
.../iceberg/spark/procedures/SparkProcedures.java | 72 -
.../iceberg/spark/source/BaseBatchReader.java | 128 -
.../apache/iceberg/spark/source/BaseReader.java | 230 --
.../apache/iceberg/spark/source/BaseRowReader.java | 125 -
.../iceberg/spark/source/BatchDataReader.java | 111 -
.../iceberg/spark/source/ChangelogRowReader.java | 173 --
.../spark/source/EqualityDeleteRowReader.java | 56 -
.../iceberg/spark/source/HasIcebergCatalog.java | 30 -
.../apache/iceberg/spark/source/IcebergSource.java | 255 --
.../iceberg/spark/source/InternalRowWrapper.java | 109 -
.../spark/source/PositionDeletesRowReader.java | 111 -
.../apache/iceberg/spark/source/RowDataReader.java | 122 -
.../spark/source/SerializableTableWithSize.java | 98 -
.../apache/iceberg/spark/source/SparkBatch.java | 181 --
.../iceberg/spark/source/SparkBatchQueryScan.java | 259 --
.../iceberg/spark/source/SparkChangelogScan.java | 176 --
.../iceberg/spark/source/SparkChangelogTable.java | 120 -
.../iceberg/spark/source/SparkCleanupUtil.java | 138 -
.../spark/source/SparkColumnarReaderFactory.java | 64 -
.../spark/source/SparkCopyOnWriteOperation.java | 107 -
.../iceberg/spark/source/SparkCopyOnWriteScan.java | 186 --
.../spark/source/SparkFileWriterFactory.java | 275 --
.../iceberg/spark/source/SparkInputPartition.java | 106 -
.../iceberg/spark/source/SparkLocalScan.java | 66 -
.../iceberg/spark/source/SparkMetadataColumn.java | 50 -
.../spark/source/SparkMicroBatchStream.java | 494 ----
.../spark/source/SparkPartitionedFanoutWriter.java | 55 -
.../spark/source/SparkPartitionedWriter.java | 55 -
.../spark/source/SparkPartitioningAwareScan.java | 282 ---
.../spark/source/SparkPositionDeletesRewrite.java | 413 ---
.../source/SparkPositionDeletesRewriteBuilder.java | 113 -
.../spark/source/SparkPositionDeltaOperation.java | 120 -
.../spark/source/SparkPositionDeltaWrite.java | 732 ------
.../source/SparkPositionDeltaWriteBuilder.java | 139 -
.../source/SparkRowLevelOperationBuilder.java | 114 -
.../spark/source/SparkRowReaderFactory.java | 68 -
.../org/apache/iceberg/spark/source/SparkScan.java | 184 --
.../iceberg/spark/source/SparkScanBuilder.java | 717 ------
.../iceberg/spark/source/SparkStagedScan.java | 102 -
.../spark/source/SparkStagedScanBuilder.java | 92 -
.../apache/iceberg/spark/source/SparkTable.java | 439 ----
.../apache/iceberg/spark/source/SparkWrite.java | 776 ------
.../iceberg/spark/source/SparkWriteBuilder.java | 269 --
.../iceberg/spark/source/StagedSparkTable.java | 41 -
.../org/apache/iceberg/spark/source/Stats.java | 42 -
.../iceberg/spark/source/StreamingOffset.java | 157 --
.../iceberg/spark/source/StructInternalRow.java | 379 ---
.../iceberg/spark/source/metrics/NumDeletes.java | 47 -
.../iceberg/spark/source/metrics/NumSplits.java | 45 -
.../spark/source/metrics/TaskNumDeletes.java | 39 -
.../spark/source/metrics/TaskNumSplits.java | 39 -
.../analysis/NoSuchProcedureException.java | 36 -
.../sql/connector/iceberg/catalog/Procedure.java | 50 -
.../iceberg/catalog/ProcedureCatalog.java | 40 -
.../iceberg/catalog/ProcedureParameter.java | 56 -
.../iceberg/catalog/ProcedureParameterImpl.java | 75 -
.../connector/iceberg/write/DeltaBatchWrite.java | 28 -
.../sql/connector/iceberg/write/DeltaWrite.java | 32 -
.../connector/iceberg/write/DeltaWriteBuilder.java | 30 -
.../sql/connector/iceberg/write/DeltaWriter.java | 57 -
.../iceberg/write/DeltaWriterFactory.java | 28 -
.../iceberg/write/ExtendedLogicalWriteInfo.java | 31 -
.../sql/connector/iceberg/write/SupportsDelta.java | 35 -
...org.apache.spark.sql.sources.DataSourceRegister | 20 -
.../expressions/TransformExpressions.scala | 163 --
.../logical/SetWriteDistributionAndOrdering.scala | 44 -
.../plans/logical/SortOrderParserUtil.scala | 40 -
.../utils/DistributionAndOrderingUtils.scala | 189 --
.../spark/sql/catalyst/utils/PlanUtils.scala | 42 -
.../datasources/SparkExpressionConverter.scala | 65 -
.../test/java/org/apache/iceberg/KryoHelpers.java | 51 -
.../java/org/apache/iceberg/TaskCheckHelper.java | 112 -
.../apache/iceberg/TestDataFileSerialization.java | 176 --
.../apache/iceberg/TestFileIOSerialization.java | 109 -
.../TestHadoopMetricsContextSerialization.java | 56 -
.../iceberg/TestManifestFileSerialization.java | 217 --
.../apache/iceberg/TestScanTaskSerialization.java | 198 --
.../org/apache/iceberg/TestTableSerialization.java | 176 --
.../java/org/apache/iceberg/ValidationHelpers.java | 78 -
.../apache/iceberg/spark/SparkCatalogConfig.java | 67 -
.../apache/iceberg/spark/SparkCatalogTestBase.java | 62 -
.../org/apache/iceberg/spark/SparkTestBase.java | 287 ---
.../iceberg/spark/SparkTestBaseWithCatalog.java | 105 -
.../apache/iceberg/spark/SparkTestHelperBase.java | 88 -
.../iceberg/spark/TestChangelogIterator.java | 359 ---
.../iceberg/spark/TestFileRewriteCoordinator.java | 273 --
.../apache/iceberg/spark/TestFunctionCatalog.java | 140 --
.../org/apache/iceberg/spark/TestSpark3Util.java | 143 --
.../iceberg/spark/TestSparkCachedTableCatalog.java | 94 -
.../iceberg/spark/TestSparkCatalogOperations.java | 97 -
.../TestSparkDistributionAndOrderingUtil.java | 2295 -----------------
.../org/apache/iceberg/spark/TestSparkFilters.java | 168 --
.../apache/iceberg/spark/TestSparkSchemaUtil.java | 84 -
.../iceberg/spark/TestSparkSessionCatalog.java | 113 -
.../apache/iceberg/spark/TestSparkTableUtil.java | 103 -
.../apache/iceberg/spark/TestSparkV2Filters.java | 267 --
.../iceberg/spark/TestSparkValueConverter.java | 94 -
.../apache/iceberg/spark/TestSparkWriteConf.java | 197 --
.../iceberg/spark/actions/TestCreateActions.java | 1056 --------
.../actions/TestDeleteReachableFilesAction.java | 389 ---
.../spark/actions/TestExpireSnapshotsAction.java | 1354 ----------
.../spark/actions/TestMigrateTableAction.java | 70 -
.../spark/actions/TestRemoveOrphanFilesAction.java | 1090 --------
.../actions/TestRemoveOrphanFilesAction3.java | 199 --
.../spark/actions/TestRewriteDataFilesAction.java | 1893 --------------
.../spark/actions/TestRewriteManifestsAction.java | 647 -----
.../TestRewritePositionDeleteFilesAction.java | 1070 --------
.../spark/actions/TestSnapshotTableAction.java | 69 -
.../spark/actions/TestSparkFileRewriter.java | 440 ----
.../apache/iceberg/spark/data/AvroDataTest.java | 609 -----
.../apache/iceberg/spark/data/GenericsHelpers.java | 324 ---
.../org/apache/iceberg/spark/data/RandomData.java | 370 ---
.../org/apache/iceberg/spark/data/TestHelpers.java | 877 -------
.../apache/iceberg/spark/data/TestOrcWrite.java | 59 -
.../iceberg/spark/data/TestParquetAvroReader.java | 236 --
.../iceberg/spark/data/TestParquetAvroWriter.java | 123 -
.../iceberg/spark/data/TestSparkAvroEnums.java | 96 -
.../iceberg/spark/data/TestSparkAvroReader.java | 72 -
.../iceberg/spark/data/TestSparkDateTimes.java | 74 -
.../data/TestSparkOrcReadMetadataColumns.java | 220 --
.../iceberg/spark/data/TestSparkOrcReader.java | 109 -
.../data/TestSparkParquetReadMetadataColumns.java | 311 ---
.../iceberg/spark/data/TestSparkParquetReader.java | 235 --
.../iceberg/spark/data/TestSparkParquetWriter.java | 119 -
.../spark/data/TestSparkRecordOrcReaderWriter.java | 155 --
.../spark/data/TestVectorizedOrcDataReader.java | 166 --
...estParquetDictionaryEncodedVectorizedReads.java | 94 -
...naryFallbackToPlainEncodingVectorizedReads.java | 74 -
.../vectorized/TestParquetVectorizedReads.java | 400 ---
.../apache/iceberg/spark/source/ComplexRecord.java | 74 -
.../spark/source/DataFrameWriteTestBase.java | 140 --
.../spark/source/FilePathLastModifiedRecord.java | 80 -
.../iceberg/spark/source/FourColumnRecord.java | 106 -
.../apache/iceberg/spark/source/LogMessage.java | 119 -
.../apache/iceberg/spark/source/ManualSource.java | 75 -
.../apache/iceberg/spark/source/NestedRecord.java | 77 -
.../apache/iceberg/spark/source/ScanTestBase.java | 126 -
.../apache/iceberg/spark/source/SimpleRecord.java | 78 -
.../spark/source/SparkSQLExecutionHelper.java | 70 -
.../spark/source/TestAvroDataFrameWrite.java | 33 -
.../apache/iceberg/spark/source/TestAvroScan.java | 58 -
.../iceberg/spark/source/TestBaseReader.java | 283 ---
.../iceberg/spark/source/TestChangelogReader.java | 259 --
.../spark/source/TestDataFrameWriterV2.java | 364 ---
.../source/TestDataFrameWriterV2Coercion.java | 69 -
.../spark/source/TestDataSourceOptions.java | 507 ----
.../iceberg/spark/source/TestFilteredScan.java | 683 -----
.../spark/source/TestForwardCompatibility.java | 227 --
.../iceberg/spark/source/TestIcebergSource.java | 42 -
.../source/TestIcebergSourceHadoopTables.java | 71 -
.../spark/source/TestIcebergSourceHiveTables.java | 86 -
.../spark/source/TestIcebergSourceTablesBase.java | 2359 -----------------
.../iceberg/spark/source/TestIcebergSpark.java | 250 --
.../spark/source/TestIdentityPartitionData.java | 209 --
.../spark/source/TestInternalRowWrapper.java | 80 -
.../source/TestMetadataTableReadableMetrics.java | 347 ---
.../TestMetadataTablesWithPartitionEvolution.java | 731 ------
.../spark/source/TestORCDataFrameWrite.java | 33 -
.../spark/source/TestParquetDataFrameWrite.java | 33 -
.../iceberg/spark/source/TestParquetScan.java | 87 -
.../spark/source/TestParquetVectorizedScan.java | 26 -
.../iceberg/spark/source/TestPartitionPruning.java | 465 ----
.../iceberg/spark/source/TestPartitionValues.java | 504 ----
.../iceberg/spark/source/TestPathIdentifier.java | 85 -
.../spark/source/TestPositionDeletesTable.java | 1568 ------------
.../iceberg/spark/source/TestReadProjection.java | 608 -----
.../TestRequiredDistributionAndOrdering.java | 329 ---
.../iceberg/spark/source/TestRuntimeFiltering.java | 472 ----
.../spark/source/TestSnapshotSelection.java | 548 ----
.../iceberg/spark/source/TestSparkAggregates.java | 76 -
.../iceberg/spark/source/TestSparkCatalog.java | 66 -
.../source/TestSparkCatalogCacheExpiration.java | 150 --
.../source/TestSparkCatalogHadoopOverrides.java | 145 --
.../iceberg/spark/source/TestSparkDataFile.java | 224 --
.../iceberg/spark/source/TestSparkDataWrite.java | 745 ------
.../spark/source/TestSparkFileWriterFactory.java | 69 -
.../spark/source/TestSparkMetadataColumns.java | 336 ---
.../spark/source/TestSparkPartitioningWriters.java | 69 -
.../source/TestSparkPositionDeltaWriters.java | 69 -
.../spark/source/TestSparkReadProjection.java | 258 --
.../spark/source/TestSparkReaderDeletes.java | 690 -----
.../source/TestSparkReaderWithBloomFilter.java | 373 ---
.../spark/source/TestSparkRollingFileWriters.java | 55 -
.../apache/iceberg/spark/source/TestSparkScan.java | 79 -
.../iceberg/spark/source/TestSparkStagedScan.java | 127 -
.../iceberg/spark/source/TestSparkTable.java | 60 -
.../spark/source/TestSparkWriterMetrics.java | 67 -
.../iceberg/spark/source/TestStreamingOffset.java | 56 -
.../spark/source/TestStructuredStreaming.java | 303 ---
.../spark/source/TestStructuredStreamingRead3.java | 672 -----
.../apache/iceberg/spark/source/TestTables.java | 209 --
.../spark/source/TestTimestampWithoutZone.java | 309 ---
.../spark/source/TestWriteMetricsConfig.java | 299 ---
.../iceberg/spark/source/ThreeColumnRecord.java | 83 -
.../spark/sql/PartitionedWritesTestBase.java | 282 ---
.../iceberg/spark/sql/TestAggregatePushDown.java | 799 ------
.../apache/iceberg/spark/sql/TestAlterTable.java | 331 ---
.../apache/iceberg/spark/sql/TestCreateTable.java | 390 ---
.../iceberg/spark/sql/TestCreateTableAsSelect.java | 415 ---
.../apache/iceberg/spark/sql/TestDeleteFrom.java | 170 --
.../apache/iceberg/spark/sql/TestDropTable.java | 156 --
.../iceberg/spark/sql/TestFilterPushDown.java | 574 -----
.../apache/iceberg/spark/sql/TestNamespaceSQL.java | 245 --
.../iceberg/spark/sql/TestPartitionedWrites.java | 29 -
.../spark/sql/TestPartitionedWritesAsSelect.java | 141 --
.../spark/sql/TestPartitionedWritesToBranch.java | 52 -
.../sql/TestPartitionedWritesToWapBranch.java | 99 -
.../apache/iceberg/spark/sql/TestRefreshTable.java | 78 -
.../org/apache/iceberg/spark/sql/TestSelect.java | 511 ----
.../iceberg/spark/sql/TestSparkBucketFunction.java | 359 ---
.../iceberg/spark/sql/TestSparkDaysFunction.java | 100 -
.../iceberg/spark/sql/TestSparkHoursFunction.java | 82 -
.../iceberg/spark/sql/TestSparkMonthsFunction.java | 116 -
.../spark/sql/TestSparkTruncateFunction.java | 479 ----
.../iceberg/spark/sql/TestSparkYearsFunction.java | 118 -
.../spark/sql/TestStoragePartitionedJoins.java | 586 -----
.../spark/sql/TestTimestampWithoutZone.java | 239 --
.../iceberg/spark/sql/TestUnpartitionedWrites.java | 29 -
.../spark/sql/TestUnpartitionedWritesToBranch.java | 62 -
.../spark/sql/UnpartitionedWritesTestBase.java | 193 --
511 files changed, 18 insertions(+), 110385 deletions(-)
diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
index 87b49e7787..d7c680e74d 100644
--- a/.github/workflows/publish-snapshot.yml
+++ b/.github/workflows/publish-snapshot.yml
@@ -41,4 +41,4 @@ jobs:
- run: |
./gradlew printVersion
./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
- ./gradlew -DflinkVersions= -DsparkVersions=3.3,3.4,3.5 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
+ ./gradlew -DflinkVersions= -DsparkVersions=3.4,3.5 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml
index e15e07fd5e..7994d23b42 100644
--- a/.github/workflows/spark-ci.yml
+++ b/.github/workflows/spark-ci.yml
@@ -70,13 +70,11 @@ jobs:
strategy:
matrix:
jvm: [11, 17, 21]
- spark: ['3.3', '3.4', '3.5']
+ spark: ['3.4', '3.5']
scala: ['2.12', '2.13']
exclude:
# Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
# Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831)
- - jvm: 21
- spark: '3.3'
- jvm: 21
spark: '3.4'
env:
diff --git a/.gitignore b/.gitignore
index e4c9e1a16a..cbe36d543f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,8 +31,6 @@ site/docs/.asf.yaml
site/docs/javadoc/
# benchmark output
-spark/v3.3/spark/benchmark/*
-spark/v3.3/spark-extensions/benchmark/*
spark/v3.4/spark/benchmark/*
spark/v3.4/spark-extensions/benchmark/*
spark/v3.5/spark/benchmark/*
diff --git a/build.gradle b/build.gradle
index d7cf13f569..099192abaf 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1080,7 +1080,6 @@ project(':iceberg-bom') {
// the BOM references the artifacts for all Scala versions.
def sparkScalaPattern = ~"(.*)-([0-9][.][0-9]+)_([0-9][.][0-9]+)"
def sparkScalaVersions = [
- "3.3": ["2.12", "2.13"],
"3.4": ["2.12", "2.13"],
"3.5": ["2.12", "2.13"],
]
diff --git a/dev/stage-binaries.sh b/dev/stage-binaries.sh
index 7a6f4e7719..43010324f9 100755
--- a/dev/stage-binaries.sh
+++ b/dev/stage-binaries.sh
@@ -20,14 +20,13 @@
SCALA_VERSION=2.12
FLINK_VERSIONS=1.18,1.19,1.20
-SPARK_VERSIONS=3.3,3.4,3.5
+SPARK_VERSIONS=3.4,3.5
KAFKA_VERSIONS=3
./gradlew -Prelease -DscalaVersion=$SCALA_VERSION -DflinkVersions=$FLINK_VERSIONS -DsparkVersions=$SPARK_VERSIONS -DkafkaVersions=$KAFKA_VERSIONS publishApachePublicationToMavenRepository
# Also publish Scala 2.13 Artifacts for versions that support it.
# Flink does not yet support 2.13 (and is largely dropping a user-facing dependency on Scala). Hive doesn't need a Scala specification.
-./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.3 :iceberg-spark:iceberg-spark-3.3_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.3_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.3_2.13:publishApachePublicationToMavenRepository
./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.4 :iceberg-spark:iceberg-spark-3.4_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.4_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.4_2.13:publishApachePublicationToMavenRepository
./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.5 :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository
diff --git a/docs/docs/aws.md b/docs/docs/aws.md
index 1a98a4d18e..b58ea33e88 100644
--- a/docs/docs/aws.md
+++ b/docs/docs/aws.md
@@ -440,7 +440,7 @@ This is turned off by default.
### S3 Tags
Custom [tags](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) can be added to S3 objects while writing and deleting.
-For example, to write S3 tags with Spark 3.3, you can start the Spark SQL shell with:
+For example, to write S3 tags with Spark 3.5, you can start the Spark SQL shell with:
```
spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket/my/key/prefix \
@@ -457,7 +457,7 @@ The property is set to `true` by default.
With the `s3.delete.tags` config, objects are tagged with the configured key-value pairs before deletion.
Users can configure tag-based object lifecycle policy at bucket level to transition objects to different tiers.
-For example, to add S3 delete tags with Spark 3.3, you can start the Spark SQL shell with:
+For example, to add S3 delete tags with Spark 3.5, you can start the Spark SQL shell with:
```
sh spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
@@ -473,7 +473,7 @@ Users can also use the catalog property `s3.delete.num-threads` to mention the n
When the catalog property `s3.write.table-tag-enabled` and `s3.write.namespace-tag-enabled` is set to `true` then the objects in S3 will be saved with tags: `iceberg.table=<table-name>` and `iceberg.namespace=<namespace-name>`.
Users can define access and data retention policy per namespace or table based on these tags.
-For example, to write table and namespace name as S3 tags with Spark 3.3, you can start the Spark SQL shell with:
+For example, to write table and namespace name as S3 tags with Spark 3.5, you can start the Spark SQL shell with:
```
sh spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://iceberg-warehouse/s3-tagging \
@@ -493,7 +493,7 @@ disaster recovery, etc.
For using cross-region access points, we need to additionally set `use-arn-region-enabled` catalog property to
`true` to enable `S3FileIO` to make cross-region calls, it's not required for same / multi-region access points.
-For example, to use S3 access-point with Spark 3.3, you can start the Spark SQL shell with:
+For example, to use S3 access-point with Spark 3.5, you can start the Spark SQL shell with:
```
spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket2/my/key/prefix \
@@ -520,7 +520,7 @@ you to fallback to using your IAM role (and its permission sets directly) to acc
is unable to authorize your S3 call. This can be done using the `s3.access-grants.fallback-to-iam` boolean catalog property. By default,
this property is set to `false`.
-For example, to add the S3 Access Grants Integration with Spark 3.3, you can start the Spark SQL shell with:
+For example, to add the S3 Access Grants Integration with Spark 3.5, you can start the Spark SQL shell with:
```
spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket2/my/key/prefix \
@@ -537,7 +537,7 @@ For more details on using S3 Access Grants, please refer to [Managing access wit
S3 Cross-Region bucket access can be turned on by setting catalog property `s3.cross-region-access-enabled` to `true`.
This is turned off by default to avoid first S3 API call increased latency.
-For example, to enable S3 Cross-Region bucket access with Spark 3.3, you can start the Spark SQL shell with:
+For example, to enable S3 Cross-Region bucket access with Spark 3.5, you can start the Spark SQL shell with:
```
spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket2/my/key/prefix \
@@ -554,7 +554,7 @@ For more details, please refer to [Cross-Region access for Amazon S3](https://do
To use S3 Acceleration, we need to set `s3.acceleration-enabled` catalog property to `true` to enable `S3FileIO` to make accelerated S3 calls.
-For example, to use S3 Acceleration with Spark 3.3, you can start the Spark SQL shell with:
+For example, to use S3 Acceleration with Spark 3.5, you can start the Spark SQL shell with:
```
spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket2/my/key/prefix \
@@ -572,7 +572,7 @@ When clients request a dual-stack endpoint, the bucket URL resolves to an IPv6 a
To use S3 Dual-stack, we need to set `s3.dualstack-enabled` catalog property to `true` to enable `S3FileIO` to make dual-stack S3 calls.
-For example, to use S3 Dual-stack with Spark 3.3, you can start the Spark SQL shell with:
+For example, to use S3 Dual-stack with Spark 3.5, you can start the Spark SQL shell with:
```
spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket2/my/key/prefix \
@@ -698,7 +698,7 @@ LIB_PATH=/usr/share/aws/aws-java-sdk/
ICEBERG_PACKAGES=(
- "iceberg-spark-runtime-3.3_2.12"
+ "iceberg-spark-runtime-3.5_2.12"
"iceberg-flink-runtime"
"iceberg-aws-bundle"
)
diff --git a/docs/docs/nessie.md b/docs/docs/nessie.md
index 96d8309538..83a142cb8d 100644
--- a/docs/docs/nessie.md
+++ b/docs/docs/nessie.md
@@ -33,17 +33,17 @@ See [Project Nessie](https://projectnessie.org) for more information on Nessie.
## Enabling Nessie Catalog
The `iceberg-nessie` module is bundled with Spark and Flink runtimes for all versions from `0.11.0`. To get started
-with Nessie (with spark-3.3) and Iceberg simply add the Iceberg runtime to your process. Eg: `spark-sql --packages
-org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:{{ icebergVersion }}`.
+with Nessie (with spark-3.5) and Iceberg simply add the Iceberg runtime to your process. Eg: `spark-sql --packages
+org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:{{ icebergVersion }}`.
## Spark SQL Extensions
Nessie SQL extensions can be used to manage the Nessie repo as shown below.
-Example for Spark 3.3 with scala 2.12:
+Example for Spark 3.5 with scala 2.12:
```
bin/spark-sql
- --packages "org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:{{ icebergVersion }},org.projectnessie.nessie-integrations:nessie-spark-extensions-3.3_2.12:{{ nessieVersion }}"
+ --packages "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:{{ icebergVersion }},org.projectnessie.nessie-integrations:nessie-spark-extensions-3.5_2.12:{{ nessieVersion }}"
--conf spark.sql.extensions="org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,org.projectnessie.spark.extensions.NessieSparkSessionExtensions"
--conf <other settings>
```
diff --git a/gradle.properties b/gradle.properties
index f6371942e8..c2c5b3f294 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -19,7 +19,7 @@ jmhIncludeRegex=.*
systemProp.defaultFlinkVersions=1.20
systemProp.knownFlinkVersions=1.18,1.19,1.20
systemProp.defaultSparkVersions=3.5
-systemProp.knownSparkVersions=3.3,3.4,3.5
+systemProp.knownSparkVersions=3.4,3.5
systemProp.defaultKafkaVersions=3
systemProp.knownKafkaVersions=3
systemProp.defaultScalaVersion=2.12
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 162060a3c6..5cb71c6cfb 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -78,7 +78,6 @@ roaringbitmap = "1.3.0"
scala-collection-compat = "2.13.0"
slf4j = "2.0.16"
snowflake-jdbc = "3.22.0"
-spark-hive33 = "3.3.4"
spark-hive34 = "3.4.4"
spark-hive35 = "3.5.4"
sqlite-jdbc = "3.49.0.0"
diff --git a/jmh.gradle b/jmh.gradle
index a5d8d62427..7503d14ff1 100644
--- a/jmh.gradle
+++ b/jmh.gradle
@@ -38,11 +38,6 @@ if (flinkVersions.contains("1.20")) {
jmhProjects.add(project(":iceberg-flink:iceberg-flink-1.20"))
}
-if (sparkVersions.contains("3.3")) {
- jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.3_${scalaVersion}"))
- jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-3.3_${scalaVersion}"))
-}
-
if (sparkVersions.contains("3.4")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.4_${scalaVersion}"))
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-3.4_${scalaVersion}"))
diff --git a/settings.gradle b/settings.gradle
index efd9336d2e..42e0e2816d 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -137,18 +137,6 @@ if (flinkVersions.contains("1.20")) {
project(":iceberg-flink:flink-runtime-1.20").name = "iceberg-flink-runtime-1.20"
}
-if (sparkVersions.contains("3.3")) {
- include ":iceberg-spark:spark-3.3_${scalaVersion}"
- include ":iceberg-spark:spark-extensions-3.3_${scalaVersion}"
- include ":iceberg-spark:spark-runtime-3.3_${scalaVersion}"
- project(":iceberg-spark:spark-3.3_${scalaVersion}").projectDir = file('spark/v3.3/spark')
- project(":iceberg-spark:spark-3.3_${scalaVersion}").name = "iceberg-spark-3.3_${scalaVersion}"
- project(":iceberg-spark:spark-extensions-3.3_${scalaVersion}").projectDir = file('spark/v3.3/spark-extensions')
- project(":iceberg-spark:spark-extensions-3.3_${scalaVersion}").name = "iceberg-spark-extensions-3.3_${scalaVersion}"
- project(":iceberg-spark:spark-runtime-3.3_${scalaVersion}").projectDir = file('spark/v3.3/spark-runtime')
- project(":iceberg-spark:spark-runtime-3.3_${scalaVersion}").name = "iceberg-spark-runtime-3.3_${scalaVersion}"
-}
-
if (sparkVersions.contains("3.4")) {
include ":iceberg-spark:spark-3.4_${scalaVersion}"
include ":iceberg-spark:spark-extensions-3.4_${scalaVersion}"
diff --git a/site/docs/multi-engine-support.md b/site/docs/multi-engine-support.md
index ab20d59ff4..984f4e40df 100644
--- a/site/docs/multi-engine-support.md
+++ b/site/docs/multi-engine-support.md
@@ -67,7 +67,7 @@ Each engine version undergoes the following lifecycle stages:
| 3.0 | End of Life | 0.9.0 | 1.0.0 | [iceberg-spark-runtime-3.0_2.12](https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.0_2.12/1.0.0/iceberg-spark-runtime-3.0_2.12-1.0.0.jar) |
| 3.1 | End of Life | 0.12.0 | 1.3.1 | [iceberg-spark-runtime-3.1_2.12](https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.1_2.12/1.3.1/iceberg-spark-runtime-3.1_2.12-1.3.1.jar) [1] |
| 3.2 | End of Life | 0.13.0 | 1.4.3 | [iceberg-spark-runtime-3.2_2.12](https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.2_2.12/1.4.3/iceberg-spark-runtime-3.2_2.12-1.4.3.jar) |
-| 3.3 | Deprecated | 0.14.0 | {{ icebergVersion }} | [iceberg-spark-runtime-3.3_2.12](https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.3_2.12/{{ icebergVersion }}/iceberg-spark-runtime-3.3_2.12-{{ icebergVersion }}.jar) |
+| 3.3 | End of Life | 0.14.0 | 1.8.0 | [iceberg-spark-runtime-3.3_2.12](https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.3_2.12/1.8.0/iceberg-spark-runtime-3.3_2.12-1.8.0.jar) |
| 3.4 | Maintained | 1.3.0 | {{ icebergVersion }} | [iceberg-spark-runtime-3.4_2.12](https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.4_2.12/{{ icebergVersion }}/iceberg-spark-runtime-3.4_2.12-{{ icebergVersion }}.jar) |
| 3.5 | Maintained | 1.4.0 | {{ icebergVersion }} | [iceberg-spark-runtime-3.5_2.12](https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/{{ icebergVersion }}/iceberg-spark-runtime-3.5_2.12-{{ icebergVersion }}.jar) |
diff --git a/spark/build.gradle b/spark/build.gradle
index c2bc5f8a14..88d2524280 100644
--- a/spark/build.gradle
+++ b/spark/build.gradle
@@ -20,10 +20,6 @@
// add enabled Spark version modules to the build
def sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",")
-if (sparkVersions.contains("3.3")) {
- apply from: file("$projectDir/v3.3/build.gradle")
-}
-
if (sparkVersions.contains("3.4")) {
apply from: file("$projectDir/v3.4/build.gradle")
}
diff --git a/spark/v3.3/build.gradle b/spark/v3.3/build.gradle
deleted file mode 100644
index c8d4b4549f..0000000000
--- a/spark/v3.3/build.gradle
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-String sparkMajorVersion = '3.3'
-String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
-
-def sparkProjects = [
- project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}"),
- project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}"),
- project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}"),
-]
-
-configure(sparkProjects) {
- configurations {
- all {
- resolutionStrategy {
- force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson213.get()}"
- force "com.fasterxml.jackson.core:jackson-databind:2.13.4.2"
- force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson213.get()}"
- }
- }
- }
-}
-
-project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
- apply plugin: 'scala'
- apply plugin: 'com.github.alisiikh.scalastyle'
-
- sourceSets {
- main {
- scala.srcDirs = ['src/main/scala', 'src/main/java']
- java.srcDirs = []
- }
- }
-
- dependencies {
- implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
- api project(':iceberg-api')
- implementation project(':iceberg-common')
- implementation project(':iceberg-core')
- implementation project(':iceberg-data')
- implementation project(':iceberg-orc')
- implementation project(':iceberg-parquet')
- implementation project(':iceberg-arrow')
- implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
-
- compileOnly libs.errorprone.annotations
- compileOnly libs.avro.avro
- compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark.hive33.get()}") {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.apache.arrow'
- exclude group: 'org.apache.parquet'
- // to make sure netty libs only come from project(':iceberg-arrow')
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- exclude group: 'org.roaringbitmap'
- }
-
- implementation libs.parquet.column
- implementation libs.parquet.hadoop
-
- implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
- exclude group: 'org.apache.hadoop'
- exclude group: 'commons-lang'
- // These artifacts are shaded and included in the orc-core fat jar
- exclude group: 'com.google.protobuf', module: 'protobuf-java'
- exclude group: 'org.apache.hive', module: 'hive-storage-api'
- }
-
- implementation(libs.arrow.vector) {
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
-
- testImplementation(libs.hadoop2.minicluster) {
- exclude group: 'org.apache.avro', module: 'avro'
- // to make sure netty libs only come from
- // project(':iceberg-arrow')
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- }
- testImplementation project(path: ':iceberg-hive-metastore')
- testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
- testImplementation libs.sqlite.jdbc
- testImplementation libs.awaitility
- testImplementation libs.junit.vintage.engine
- }
-
- test {
- useJUnitPlatform()
- }
-
- tasks.withType(Test) {
- // Vectorized reads need more memory
- maxHeapSize '2560m'
- }
-}
-
-project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") {
- apply plugin: 'java-library'
- apply plugin: 'scala'
- apply plugin: 'com.github.alisiikh.scalastyle'
- apply plugin: 'antlr'
-
- configurations {
- /*
- The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This
- bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate
- runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with
- the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add
- the runtime dependency back to only the runtime configuration manually.
- */
- implementation {
- extendsFrom = extendsFrom.findAll { it != configurations.antlr }
- }
- }
-
- dependencies {
- implementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
- implementation libs.roaringbitmap
-
- compileOnly "org.scala-lang:scala-library"
- compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
- compileOnly project(':iceberg-api')
- compileOnly project(':iceberg-core')
- compileOnly project(':iceberg-common')
- compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
- compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark.hive33.get()}") {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.apache.arrow'
- exclude group: 'org.apache.parquet'
- // to make sure netty libs only come from project(':iceberg-arrow')
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- exclude group: 'org.roaringbitmap'
- }
- compileOnly libs.errorprone.annotations
-
- testImplementation project(path: ':iceberg-data')
- testImplementation project(path: ':iceberg-parquet')
- testImplementation project(path: ':iceberg-hive-metastore')
- testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
-
- testImplementation libs.avro.avro
- testImplementation libs.parquet.hadoop
- testImplementation libs.junit.vintage.engine
-
- // Required because we remove antlr plugin dependencies from the compile configuration, see note above
- runtimeOnly libs.antlr.runtime
- antlr libs.antlr.antlr4
- }
-
- generateGrammarSource {
- maxHeapSize = "64m"
- arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions']
- }
-}
-
-project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}") {
- apply plugin: 'com.gradleup.shadow'
-
- tasks.jar.dependsOn tasks.shadowJar
-
- sourceSets {
- integration {
- java.srcDir "$projectDir/src/integration/java"
- resources.srcDir "$projectDir/src/integration/resources"
- }
- }
-
- configurations {
- implementation {
- exclude group: 'org.apache.spark'
- // included in Spark
- exclude group: 'org.slf4j'
- exclude group: 'org.apache.commons'
- exclude group: 'commons-pool'
- exclude group: 'commons-codec'
- exclude group: 'org.xerial.snappy'
- exclude group: 'javax.xml.bind'
- exclude group: 'javax.annotation'
- exclude group: 'com.github.luben'
- exclude group: 'com.ibm.icu'
- exclude group: 'org.glassfish'
- exclude group: 'org.abego.treelayout'
- exclude group: 'org.antlr'
- exclude group: 'org.scala-lang'
- exclude group: 'org.scala-lang.modules'
- }
- }
-
- dependencies {
- api project(':iceberg-api')
- implementation project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
- implementation project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
- implementation project(':iceberg-aws')
- implementation project(':iceberg-azure')
- implementation(project(':iceberg-aliyun')) {
- exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
- exclude group: 'org.apache.httpcomponents', module: 'httpclient'
- exclude group: 'commons-logging', module: 'commons-logging'
- }
- implementation project(':iceberg-gcp')
- implementation project(':iceberg-hive-metastore')
- implementation(project(':iceberg-nessie')) {
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
- implementation (project(':iceberg-snowflake')) {
- exclude group: 'net.snowflake' , module: 'snowflake-jdbc'
- }
-
- integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
- integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark.hive33.get()}"
- integrationImplementation libs.junit.vintage.engine
- integrationImplementation libs.slf4j.simple
- integrationImplementation libs.assertj.core
- integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
- integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
- // Not allowed on our classpath, only the runtime jar is allowed
- integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
- integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
- integrationCompileOnly project(':iceberg-api')
- }
-
- shadowJar {
- configurations = [project.configurations.runtimeClasspath]
-
- zip64 true
-
- // include the LICENSE and NOTICE files for the shaded Jar
- from(projectDir) {
- include 'LICENSE'
- include 'NOTICE'
- }
-
- // Relocate dependencies to avoid conflicts
- relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
- relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
- relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
- relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
- relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
- relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
- relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
- relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
- relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
- relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
- relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
- relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
- relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
- relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
- // relocate Arrow and related deps to shade Iceberg specific version
- relocate 'io.netty', 'org.apache.iceberg.shaded.io.netty'
- relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow'
- relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch'
- relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
- relocate 'org.roaringbitmap', 'org.apache.iceberg.shaded.org.roaringbitmap'
-
- archiveClassifier.set(null)
- }
-
- task integrationTest(type: Test) {
- description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
- group = "verification"
- jvmArgs += project.property('extraJvmArgs')
- testClassesDirs = sourceSets.integration.output.classesDirs
- classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
- inputs.file(shadowJar.archiveFile.get().asFile.path)
- }
- integrationTest.dependsOn shadowJar
- check.dependsOn integrationTest
-
- jar {
- enabled = false
- }
-}
-
diff --git a/spark/v3.3/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java b/spark/v3.3/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
deleted file mode 100644
index 2ea3153576..0000000000
--- a/spark/v3.3/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark;
-
-import static org.apache.spark.sql.functions.current_date;
-import static org.apache.spark.sql.functions.date_add;
-import static org.apache.spark.sql.functions.expr;
-
-import com.google.errorprone.annotations.FormatMethod;
-import com.google.errorprone.annotations.FormatString;
-import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.DistributionMode;
-import org.apache.iceberg.RowLevelOperationMode;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.catalyst.parser.ParseException;
-import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.sql.types.StructType;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Warmup;
-
-/**
- * A benchmark that evaluates the performance of the cardinality check in MERGE operations.
- *
- * <p>To run this benchmark for spark-3.3: <code>
- * ./gradlew -DsparkVersions=3.3 :iceberg-spark:iceberg-spark-extensions-3.3_2.12:jmh
- * -PjmhIncludeRegex=MergeCardinalityCheckBenchmark
- * -PjmhOutputPath=benchmark/iceberg-merge-cardinality-check-benchmark.txt
- * </code>
- */
-@Fork(1)
-@State(Scope.Benchmark)
-@Warmup(iterations = 3)
-@Measurement(iterations = 5)
-@BenchmarkMode(Mode.SingleShotTime)
-public class MergeCardinalityCheckBenchmark {
-
- private static final String TABLE_NAME = "test_table";
- private static final int NUM_FILES = 5;
- private static final int NUM_ROWS_PER_FILE = 1_000_000;
- private static final int NUM_UNMATCHED_RECORDS_PER_MERGE = 100_000;
-
- private final Configuration hadoopConf = new Configuration();
- private SparkSession spark;
- private long originalSnapshotId;
-
- @Setup
- public void setupBenchmark() throws NoSuchTableException, ParseException {
- setupSpark();
- initTable();
- appendData();
-
- Table table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
- this.originalSnapshotId = table.currentSnapshot().snapshotId();
- }
-
- @TearDown
- public void tearDownBenchmark() {
- tearDownSpark();
- dropTable();
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteMergeCardinalityCheck10PercentUpdates() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.1);
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteMergeCardinalityCheck30PercentUpdates() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.3);
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteMergeCardinalityCheck90PercentUpdates() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.9);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadMergeCardinalityCheck10PercentUpdates() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.1);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadMergeCardinalityCheck30PercentUpdates() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.3);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadMergeCardinalityCheck90PercentUpdates() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.9);
- }
-
- private void runBenchmark(RowLevelOperationMode mode, double updatePercentage) {
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')",
- TABLE_NAME, TableProperties.MERGE_MODE, mode.modeName());
-
- Dataset<Long> insertDataDF = spark.range(-NUM_UNMATCHED_RECORDS_PER_MERGE, 0, 1);
- Dataset<Long> updateDataDF = spark.range((long) (updatePercentage * NUM_ROWS_PER_FILE));
- Dataset<Long> sourceDF = updateDataDF.union(insertDataDF);
- sourceDF.createOrReplaceTempView("source");
-
- sql(
- "MERGE INTO %s t USING source s "
- + "ON t.id = s.id "
- + "WHEN MATCHED THEN "
- + " UPDATE SET stringCol = 'invalid' "
- + "WHEN NOT MATCHED THEN "
- + " INSERT (id, intCol, floatCol, doubleCol, decimalCol, dateCol, timestampCol, stringCol) "
- + " VALUES (s.id, null, null, null, null, null, null, 'new')",
- TABLE_NAME);
-
- sql(
- "CALL system.rollback_to_snapshot(table => '%s', snapshot_id => %dL)",
- TABLE_NAME, originalSnapshotId);
- }
-
- private void setupSpark() {
- this.spark =
- SparkSession.builder()
- .config("spark.ui.enabled", false)
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
- .config("spark.sql.catalog.spark_catalog.type", "hadoop")
- .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
- .config(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key(), "false")
- .config(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false")
- .config(SQLConf.SHUFFLE_PARTITIONS().key(), "2")
- .master("local")
- .getOrCreate();
- }
-
- private void tearDownSpark() {
- spark.stop();
- }
-
- private void initTable() {
- sql(
- "CREATE TABLE %s ( "
- + " id LONG, intCol INT, floatCol FLOAT, doubleCol DOUBLE, "
- + " decimalCol DECIMAL(20, 5), dateCol DATE, timestampCol TIMESTAMP, "
- + " stringCol STRING)"
- + "USING iceberg "
- + "TBLPROPERTIES ("
- + " '%s' '%s',"
- + " '%s' '%d',"
- + " '%s' '%d')",
- TABLE_NAME,
- TableProperties.MERGE_DISTRIBUTION_MODE,
- DistributionMode.NONE.modeName(),
- TableProperties.SPLIT_OPEN_FILE_COST,
- Integer.MAX_VALUE,
- TableProperties.FORMAT_VERSION,
- 2);
-
- sql("ALTER TABLE %s WRITE ORDERED BY id", TABLE_NAME);
- }
-
- private void dropTable() {
- sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
- }
-
- private void appendData() throws NoSuchTableException {
- for (int fileNum = 1; fileNum <= NUM_FILES; fileNum++) {
- Dataset<Row> inputDF =
- spark
- .range(NUM_ROWS_PER_FILE)
- .withColumn("intCol", expr("CAST(id AS INT)"))
- .withColumn("floatCol", expr("CAST(id AS FLOAT)"))
- .withColumn("doubleCol", expr("CAST(id AS DOUBLE)"))
- .withColumn("decimalCol", expr("CAST(id AS DECIMAL(20, 5))"))
- .withColumn("dateCol", date_add(current_date(), fileNum))
- .withColumn("timestampCol", expr("TO_TIMESTAMP(dateCol)"))
- .withColumn("stringCol", expr("CAST(dateCol AS STRING)"));
- appendAsFile(inputDF);
- }
- }
-
- private void appendAsFile(Dataset<Row> df) throws NoSuchTableException {
- // ensure the schema is precise (including nullability)
- StructType sparkSchema = spark.table(TABLE_NAME).schema();
- spark.createDataFrame(df.rdd(), sparkSchema).coalesce(1).writeTo(TABLE_NAME).append();
- }
-
- private String newWarehouseDir() {
- return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
- }
-
- @FormatMethod
- private void sql(@FormatString String query, Object... args) {
- spark.sql(String.format(query, args));
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 b/spark/v3.3/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
deleted file mode 100644
index b962699d9b..0000000000
--- a/spark/v3.3/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * This file is an adaptation of Presto's and Spark's grammar files.
- */
-
-grammar IcebergSqlExtensions;
-
-@lexer::members {
- /**
- * Verify whether current token is a valid decimal token (which contains dot).
- * Returns true if the character that follows the token is not a digit or letter or underscore.
- *
- * For example:
- * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'.
- * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'.
- * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'.
- * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed
- * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+'
- * which is not a digit or letter or underscore.
- */
- public boolean isValidDecimal() {
- int nextChar = _input.LA(1);
- if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' ||
- nextChar == '_') {
- return false;
- } else {
- return true;
- }
- }
-
- /**
- * This method will be called when we see '/*' and try to match it as a bracketed comment.
- * If the next character is '+', it should be parsed as hint later, and we cannot match
- * it as a bracketed comment.
- *
- * Returns true if the next character is '+'.
- */
- public boolean isHint() {
- int nextChar = _input.LA(1);
- if (nextChar == '+') {
- return true;
- } else {
- return false;
- }
- }
-}
-
-singleStatement
- : statement EOF
- ;
-
-statement
- : CALL multipartIdentifier '(' (callArgument (',' callArgument)*)? ')' #call
- | ALTER TABLE multipartIdentifier ADD PARTITION FIELD transform (AS name=identifier)? #addPartitionField
- | ALTER TABLE multipartIdentifier DROP PARTITION FIELD transform #dropPartitionField
- | ALTER TABLE multipartIdentifier REPLACE PARTITION FIELD transform WITH transform (AS name=identifier)? #replacePartitionField
- | ALTER TABLE multipartIdentifier WRITE writeSpec #setWriteDistributionAndOrdering
- | ALTER TABLE multipartIdentifier SET IDENTIFIER_KW FIELDS fieldList #setIdentifierFields
- | ALTER TABLE multipartIdentifier DROP IDENTIFIER_KW FIELDS fieldList #dropIdentifierFields
- | ALTER TABLE multipartIdentifier createReplaceBranchClause #createOrReplaceBranch
- | ALTER TABLE multipartIdentifier createReplaceTagClause #createOrReplaceTag
- | ALTER TABLE multipartIdentifier DROP BRANCH (IF EXISTS)? identifier #dropBranch
- | ALTER TABLE multipartIdentifier DROP TAG (IF EXISTS)? identifier #dropTag
- ;
-
-createReplaceTagClause
- : (CREATE OR)? REPLACE TAG identifier tagOptions
- | CREATE TAG (IF NOT EXISTS)? identifier tagOptions
- ;
-
-createReplaceBranchClause
- : (CREATE OR)? REPLACE BRANCH identifier branchOptions
- | CREATE BRANCH (IF NOT EXISTS)? identifier branchOptions
- ;
-
-tagOptions
- : (AS OF VERSION snapshotId)? (refRetain)?
- ;
-
-branchOptions
- : (AS OF VERSION snapshotId)? (refRetain)? (snapshotRetention)?
- ;
-
-snapshotRetention
- : WITH SNAPSHOT RETENTION minSnapshotsToKeep
- | WITH SNAPSHOT RETENTION maxSnapshotAge
- | WITH SNAPSHOT RETENTION minSnapshotsToKeep maxSnapshotAge
- ;
-
-refRetain
- : RETAIN number timeUnit
- ;
-
-maxSnapshotAge
- : number timeUnit
- ;
-
-minSnapshotsToKeep
- : number SNAPSHOTS
- ;
-
-writeSpec
- : (writeDistributionSpec | writeOrderingSpec)*
- ;
-
-writeDistributionSpec
- : DISTRIBUTED BY PARTITION
- ;
-
-writeOrderingSpec
- : LOCALLY? ORDERED BY order
- | UNORDERED
- ;
-
-callArgument
- : expression #positionalArgument
- | identifier '=>' expression #namedArgument
- ;
-
-singleOrder
- : order EOF
- ;
-
-order
- : fields+=orderField (',' fields+=orderField)*
- | '(' fields+=orderField (',' fields+=orderField)* ')'
- ;
-
-orderField
- : transform direction=(ASC | DESC)? (NULLS nullOrder=(FIRST | LAST))?
- ;
-
-transform
- : multipartIdentifier #identityTransform
- | transformName=identifier
- '(' arguments+=transformArgument (',' arguments+=transformArgument)* ')' #applyTransform
- ;
-
-transformArgument
- : multipartIdentifier
- | constant
- ;
-
-expression
- : constant
- | stringMap
- | stringArray
- ;
-
-constant
- : number #numericLiteral
- | booleanValue #booleanLiteral
- | STRING+ #stringLiteral
- | identifier STRING #typeConstructor
- ;
-
-stringMap
- : MAP '(' constant (',' constant)* ')'
- ;
-
-stringArray
- : ARRAY '(' constant (',' constant)* ')'
- ;
-
-booleanValue
- : TRUE | FALSE
- ;
-
-number
- : MINUS? EXPONENT_VALUE #exponentLiteral
- | MINUS? DECIMAL_VALUE #decimalLiteral
- | MINUS? INTEGER_VALUE #integerLiteral
- | MINUS? BIGINT_LITERAL #bigIntLiteral
- | MINUS? SMALLINT_LITERAL #smallIntLiteral
- | MINUS? TINYINT_LITERAL #tinyIntLiteral
- | MINUS? DOUBLE_LITERAL #doubleLiteral
- | MINUS? FLOAT_LITERAL #floatLiteral
- | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral
- ;
-
-multipartIdentifier
- : parts+=identifier ('.' parts+=identifier)*
- ;
-
-identifier
- : IDENTIFIER #unquotedIdentifier
- | quotedIdentifier #quotedIdentifierAlternative
- | nonReserved #unquotedIdentifier
- ;
-
-quotedIdentifier
- : BACKQUOTED_IDENTIFIER
- ;
-
-fieldList
- : fields+=multipartIdentifier (',' fields+=multipartIdentifier)*
- ;
-
-nonReserved
- : ADD | ALTER | AS | ASC | BRANCH | BY | CALL | CREATE | DAYS | DESC | DROP | EXISTS | FIELD | FIRST | HOURS | IF | LAST | NOT | NULLS | OF | OR | ORDERED | PARTITION | TABLE | WRITE
- | DISTRIBUTED | LOCALLY | MINUTES | MONTHS | UNORDERED | REPLACE | RETAIN | VERSION | WITH | IDENTIFIER_KW | FIELDS | SET | SNAPSHOT | SNAPSHOTS
- | TAG | TRUE | FALSE
- | MAP
- ;
-
-snapshotId
- : number
- ;
-
-numSnapshots
- : number
- ;
-
-timeUnit
- : DAYS
- | HOURS
- | MINUTES
- ;
-
-ADD: 'ADD';
-ALTER: 'ALTER';
-AS: 'AS';
-ASC: 'ASC';
-BRANCH: 'BRANCH';
-BY: 'BY';
-CALL: 'CALL';
-DAYS: 'DAYS';
-DESC: 'DESC';
-DISTRIBUTED: 'DISTRIBUTED';
-DROP: 'DROP';
-EXISTS: 'EXISTS';
-FIELD: 'FIELD';
-FIELDS: 'FIELDS';
-FIRST: 'FIRST';
-HOURS: 'HOURS';
-IF : 'IF';
-LAST: 'LAST';
-LOCALLY: 'LOCALLY';
-MINUTES: 'MINUTES';
-MONTHS: 'MONTHS';
-CREATE: 'CREATE';
-NOT: 'NOT';
-NULLS: 'NULLS';
-OF: 'OF';
-OR: 'OR';
-ORDERED: 'ORDERED';
-PARTITION: 'PARTITION';
-REPLACE: 'REPLACE';
-RETAIN: 'RETAIN';
-RETENTION: 'RETENTION';
-IDENTIFIER_KW: 'IDENTIFIER';
-SET: 'SET';
-SNAPSHOT: 'SNAPSHOT';
-SNAPSHOTS: 'SNAPSHOTS';
-TABLE: 'TABLE';
-TAG: 'TAG';
-UNORDERED: 'UNORDERED';
-VERSION: 'VERSION';
-WITH: 'WITH';
-WRITE: 'WRITE';
-
-TRUE: 'TRUE';
-FALSE: 'FALSE';
-
-MAP: 'MAP';
-ARRAY: 'ARRAY';
-
-PLUS: '+';
-MINUS: '-';
-
-STRING
- : '\'' ( ~('\''|'\\') | ('\\' .) )* '\''
- | '"' ( ~('"'|'\\') | ('\\' .) )* '"'
- ;
-
-BIGINT_LITERAL
- : DIGIT+ 'L'
- ;
-
-SMALLINT_LITERAL
- : DIGIT+ 'S'
- ;
-
-TINYINT_LITERAL
- : DIGIT+ 'Y'
- ;
-
-INTEGER_VALUE
- : DIGIT+
- ;
-
-EXPONENT_VALUE
- : DIGIT+ EXPONENT
- | DECIMAL_DIGITS EXPONENT {isValidDecimal()}?
- ;
-
-DECIMAL_VALUE
- : DECIMAL_DIGITS {isValidDecimal()}?
- ;
-
-FLOAT_LITERAL
- : DIGIT+ EXPONENT? 'F'
- | DECIMAL_DIGITS EXPONENT? 'F' {isValidDecimal()}?
- ;
-
-DOUBLE_LITERAL
- : DIGIT+ EXPONENT? 'D'
- | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}?
- ;
-
-BIGDECIMAL_LITERAL
- : DIGIT+ EXPONENT? 'BD'
- | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}?
- ;
-
-IDENTIFIER
- : (LETTER | DIGIT | '_')+
- ;
-
-BACKQUOTED_IDENTIFIER
- : '`' ( ~'`' | '``' )* '`'
- ;
-
-fragment DECIMAL_DIGITS
- : DIGIT+ '.' DIGIT*
- | '.' DIGIT+
- ;
-
-fragment EXPONENT
- : 'E' [+-]? DIGIT+
- ;
-
-fragment DIGIT
- : [0-9]
- ;
-
-fragment LETTER
- : [A-Z]
- ;
-
-SIMPLE_COMMENT
- : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
- ;
-
-BRACKETED_COMMENT
- : '/*' {!isHint()}? (BRACKETED_COMMENT|.)*? '*/' -> channel(HIDDEN)
- ;
-
-WS
- : [ \r\n\t]+ -> channel(HIDDEN)
- ;
-
-// Catch-all for anything we can't recognize.
-// We use this to be able to ignore and recover all the text
-// when splitting statements with DelimiterLexer
-UNRECOGNIZED
- : .
- ;
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
deleted file mode 100644
index 4fb9a48a3e..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iceberg.spark.extensions
-
-import org.apache.spark.sql.SparkSessionExtensions
-import org.apache.spark.sql.catalyst.analysis.AlignedRowLevelIcebergCommandCheck
-import org.apache.spark.sql.catalyst.analysis.AlignRowLevelCommandAssignments
-import org.apache.spark.sql.catalyst.analysis.CheckMergeIntoTableConditions
-import org.apache.spark.sql.catalyst.analysis.MergeIntoIcebergTableResolutionCheck
-import org.apache.spark.sql.catalyst.analysis.ProcedureArgumentCoercion
-import org.apache.spark.sql.catalyst.analysis.ResolveMergeIntoTableReferences
-import org.apache.spark.sql.catalyst.analysis.ResolveProcedures
-import org.apache.spark.sql.catalyst.analysis.RewriteDeleteFromIcebergTable
-import org.apache.spark.sql.catalyst.analysis.RewriteMergeIntoTable
-import org.apache.spark.sql.catalyst.analysis.RewriteUpdateTable
-import org.apache.spark.sql.catalyst.optimizer.ExtendedReplaceNullWithFalseInPredicate
-import org.apache.spark.sql.catalyst.optimizer.ExtendedSimplifyConditionalsInPredicate
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser
-import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy
-import org.apache.spark.sql.execution.datasources.v2.ExtendedV2Writes
-import org.apache.spark.sql.execution.datasources.v2.OptimizeMetadataOnlyDeleteFromIcebergTable
-import org.apache.spark.sql.execution.datasources.v2.ReplaceRewrittenRowLevelCommand
-import org.apache.spark.sql.execution.datasources.v2.RowLevelCommandScanRelationPushDown
-import org.apache.spark.sql.execution.dynamicpruning.RowLevelCommandDynamicPruning
-
-class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
-
- override def apply(extensions: SparkSessionExtensions): Unit = {
- // parser extensions
- extensions.injectParser { case (_, parser) => new IcebergSparkSqlExtensionsParser(parser) }
-
- // analyzer extensions
- extensions.injectResolutionRule { spark => ResolveProcedures(spark) }
- extensions.injectResolutionRule { spark => ResolveMergeIntoTableReferences(spark) }
- extensions.injectResolutionRule { _ => CheckMergeIntoTableConditions }
- extensions.injectResolutionRule { _ => ProcedureArgumentCoercion }
- extensions.injectResolutionRule { _ => AlignRowLevelCommandAssignments }
- extensions.injectResolutionRule { _ => RewriteDeleteFromIcebergTable }
- extensions.injectResolutionRule { _ => RewriteUpdateTable }
- extensions.injectResolutionRule { _ => RewriteMergeIntoTable }
- extensions.injectCheckRule { _ => MergeIntoIcebergTableResolutionCheck }
- extensions.injectCheckRule { _ => AlignedRowLevelIcebergCommandCheck }
-
- // optimizer extensions
- extensions.injectOptimizerRule { _ => ExtendedSimplifyConditionalsInPredicate }
- extensions.injectOptimizerRule { _ => ExtendedReplaceNullWithFalseInPredicate }
- // pre-CBO rules run only once and the order of the rules is important
- // - metadata deletes have to be attempted immediately after the operator optimization
- // - dynamic filters should be added before replacing commands with rewrite plans
- // - scans must be planned before building writes
- extensions.injectPreCBORule { _ => OptimizeMetadataOnlyDeleteFromIcebergTable }
- extensions.injectPreCBORule { _ => RowLevelCommandScanRelationPushDown }
- extensions.injectPreCBORule { _ => ExtendedV2Writes }
- extensions.injectPreCBORule { spark => RowLevelCommandDynamicPruning(spark) }
- extensions.injectPreCBORule { _ => ReplaceRewrittenRowLevelCommand }
-
- // planner extensions
- extensions.injectPlannerStrategy { spark => ExtendedDataSourceV2Strategy(spark) }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala
deleted file mode 100644
index fb654b6467..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst
-
-import org.apache.spark.sql.catalyst.util.ArrayData
-import org.apache.spark.sql.catalyst.util.MapData
-import org.apache.spark.sql.types.DataType
-import org.apache.spark.sql.types.Decimal
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.unsafe.types.CalendarInterval
-import org.apache.spark.unsafe.types.UTF8String
-
-/**
- * An InternalRow that projects particular columns from another InternalRow without copying
- * the underlying data.
- */
-case class ProjectingInternalRow(schema: StructType, colOrdinals: Seq[Int]) extends InternalRow {
- assert(schema.size == colOrdinals.size)
-
- private var row: InternalRow = _
-
- override def numFields: Int = colOrdinals.size
-
- def project(row: InternalRow): Unit = {
- this.row = row
- }
-
- override def setNullAt(i: Int): Unit = {
- throw new UnsupportedOperationException("Cannot modify InternalRowProjection")
- }
-
- override def update(i: Int, value: Any): Unit = {
- throw new UnsupportedOperationException("Cannot modify InternalRowProjection")
- }
-
- override def copy(): InternalRow = {
- val newRow = if (row != null) row.copy() else null
- val newProjection = ProjectingInternalRow(schema, colOrdinals)
- newProjection.project(newRow)
- newProjection
- }
-
- override def isNullAt(ordinal: Int): Boolean = {
- row.isNullAt(colOrdinals(ordinal))
- }
-
- override def getBoolean(ordinal: Int): Boolean = {
- row.getBoolean(colOrdinals(ordinal))
- }
-
- override def getByte(ordinal: Int): Byte = {
- row.getByte(colOrdinals(ordinal))
- }
-
- override def getShort(ordinal: Int): Short = {
- row.getShort(colOrdinals(ordinal))
- }
-
- override def getInt(ordinal: Int): Int = {
- row.getInt(colOrdinals(ordinal))
- }
-
- override def getLong(ordinal: Int): Long = {
- row.getLong(colOrdinals(ordinal))
- }
-
- override def getFloat(ordinal: Int): Float = {
- row.getFloat(colOrdinals(ordinal))
- }
-
- override def getDouble(ordinal: Int): Double = {
- row.getDouble(colOrdinals(ordinal))
- }
-
- override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = {
- row.getDecimal(colOrdinals(ordinal), precision, scale)
- }
-
- override def getUTF8String(ordinal: Int): UTF8String = {
- row.getUTF8String(colOrdinals(ordinal))
- }
-
- override def getBinary(ordinal: Int): Array[Byte] = {
- row.getBinary(colOrdinals(ordinal))
- }
-
- override def getInterval(ordinal: Int): CalendarInterval = {
- row.getInterval(colOrdinals(ordinal))
- }
-
- override def getStruct(ordinal: Int, numFields: Int): InternalRow = {
- row.getStruct(colOrdinals(ordinal), numFields)
- }
-
- override def getArray(ordinal: Int): ArrayData = {
- row.getArray(colOrdinals(ordinal))
- }
-
- override def getMap(ordinal: Int): MapData = {
- row.getMap(colOrdinals(ordinal))
- }
-
- override def get(ordinal: Int, dataType: DataType): AnyRef = {
- row.get(colOrdinals(ordinal), dataType)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala
deleted file mode 100644
index ad416f2a1c..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
-import org.apache.spark.sql.catalyst.plans.logical.Assignment
-import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
-import org.apache.spark.sql.catalyst.plans.logical.InsertAction
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
-import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
-import org.apache.spark.sql.catalyst.rules.Rule
-
-/**
- * A rule that aligns assignments in UPDATE and MERGE operations.
- *
- * Note that this rule must be run before rewriting row-level commands.
- */
-object AlignRowLevelCommandAssignments
- extends Rule[LogicalPlan] with AssignmentAlignmentSupport {
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case u: UpdateIcebergTable if u.resolved && !u.aligned =>
- u.copy(assignments = alignAssignments(u.table, u.assignments))
-
- case m: MergeIntoIcebergTable if m.resolved && !m.aligned =>
- val alignedMatchedActions = m.matchedActions.map {
- case u @ UpdateAction(_, assignments) =>
- u.copy(assignments = alignAssignments(m.targetTable, assignments))
- case d: DeleteAction =>
- d
- case _ =>
- throw new AnalysisException("Matched actions can only contain UPDATE or DELETE")
- }
-
- val alignedNotMatchedActions = m.notMatchedActions.map {
- case i @ InsertAction(_, assignments) =>
- // check no nested columns are present
- val refs = assignments.map(_.key).map(AssignmentUtils.toAssignmentRef)
- refs.foreach { ref =>
- if (ref.size > 1) {
- throw new AnalysisException(
- "Nested fields are not supported inside INSERT clauses of MERGE operations: " +
- s"${ref.mkString("`", "`.`", "`")}")
- }
- }
-
- val colNames = refs.map(_.head)
-
- // check there are no duplicates
- val duplicateColNames = colNames.groupBy(identity).collect {
- case (name, matchingNames) if matchingNames.size > 1 => name
- }
-
- if (duplicateColNames.nonEmpty) {
- throw new AnalysisException(
- s"Duplicate column names inside INSERT clause: ${duplicateColNames.mkString(", ")}")
- }
-
- // reorder assignments by the target table column order
- val assignmentMap = colNames.zip(assignments).toMap
- i.copy(assignments = alignInsertActionAssignments(m.targetTable, assignmentMap))
-
- case _ =>
- throw new AnalysisException("Not matched actions can only contain INSERT")
- }
-
- m.copy(matchedActions = alignedMatchedActions, notMatchedActions = alignedNotMatchedActions)
- }
-
- private def alignInsertActionAssignments(
- targetTable: LogicalPlan,
- assignmentMap: Map[String, Assignment]): Seq[Assignment] = {
-
- val resolver = conf.resolver
-
- targetTable.output.map { targetAttr =>
- val assignment = assignmentMap
- .find { case (name, _) => resolver(name, targetAttr.name) }
- .map { case (_, assignment) => assignment }
-
- if (assignment.isEmpty) {
- throw new AnalysisException(
- s"Cannot find column '${targetAttr.name}' of the target table among " +
- s"the INSERT columns: ${assignmentMap.keys.mkString(", ")}. " +
- "INSERT clauses must provide values for all columns of the target table.")
- }
-
- val key = assignment.get.key
- val value = castIfNeeded(targetAttr, assignment.get.value, resolver)
- AssignmentUtils.handleCharVarcharLimits(Assignment(key, value))
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala
deleted file mode 100644
index d915e4f109..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
-
-object AlignedRowLevelIcebergCommandCheck extends (LogicalPlan => Unit) {
-
- override def apply(plan: LogicalPlan): Unit = {
- plan foreach {
- case m: MergeIntoIcebergTable if !m.aligned =>
- throw new AnalysisException(s"Could not align Iceberg MERGE INTO: $m")
- case u: UpdateIcebergTable if !u.aligned =>
- throw new AnalysisException(s"Could not align Iceberg UPDATE: $u")
- case _ => // OK
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala
deleted file mode 100644
index 14115bd3cb..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.SQLConfHelper
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.expressions.AnsiCast
-import org.apache.spark.sql.catalyst.expressions.AssignmentUtils._
-import org.apache.spark.sql.catalyst.expressions.Cast
-import org.apache.spark.sql.catalyst.expressions.CreateNamedStruct
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.GetStructField
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.NamedExpression
-import org.apache.spark.sql.catalyst.plans.logical.Assignment
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
-import org.apache.spark.sql.types.DataType
-import org.apache.spark.sql.types.StructField
-import org.apache.spark.sql.types.StructType
-import scala.collection.compat.immutable.ArraySeq
-import scala.collection.mutable
-
-trait AssignmentAlignmentSupport extends CastSupport {
-
- self: SQLConfHelper =>
-
- private case class ColumnUpdate(ref: Seq[String], expr: Expression)
-
- /**
- * Aligns assignments to match table columns.
- * <p>
- * This method processes and reorders given assignments so that each target column gets
- * an expression it should be set to. If a column does not have a matching assignment,
- * it will be set to its current value. For example, if one passes a table with columns c1, c2
- * and an assignment c2 = 1, this method will return c1 = c1, c2 = 1.
- * <p>
- * This method also handles updates to nested columns. If there is an assignment to a particular
- * nested field, this method will construct a new struct with one field updated
- * preserving other fields that have not been modified. For example, if one passes a table with
- * columns c1, c2 where c2 is a struct with fields n1 and n2 and an assignment c2.n2 = 1,
- * this method will return c1 = c1, c2 = struct(c2.n1, 1).
- *
- * @param table a target table
- * @param assignments assignments to align
- * @return aligned assignments that match table columns
- */
- protected def alignAssignments(
- table: LogicalPlan,
- assignments: Seq[Assignment]): Seq[Assignment] = {
-
- val columnUpdates = assignments.map(a => ColumnUpdate(toAssignmentRef(a.key), a.value))
- val outputExprs = applyUpdates(table.output, columnUpdates)
- outputExprs.zip(table.output).map {
- case (expr, attr) => handleCharVarcharLimits(Assignment(attr, expr))
- }
- }
-
- private def applyUpdates(
- cols: Seq[NamedExpression],
- updates: Seq[ColumnUpdate],
- resolver: Resolver = conf.resolver,
- namePrefix: Seq[String] = Nil): Seq[Expression] = {
-
- // iterate through columns at the current level and find which column updates match
- cols.map { col =>
- // find matches for this column or any of its children
- val prefixMatchedUpdates = updates.filter(a => resolver(a.ref.head, col.name))
- prefixMatchedUpdates match {
- // if there is no exact match and no match for children, return the column as is
- case updates if updates.isEmpty =>
- col
-
- // if there is an exact match, return the assigned expression
- case Seq(update) if isExactMatch(update, col, resolver) =>
- castIfNeeded(col, update.expr, resolver)
-
- // if there are matches only for children
- case updates if !hasExactMatch(updates, col, resolver) =>
- col.dataType match {
- case StructType(fields) =>
- // build field expressions
- val fieldExprs = fields.zipWithIndex.map { case (field, ordinal) =>
- Alias(GetStructField(col, ordinal, Some(field.name)), field.name)()
- }
-
- // recursively apply this method on nested fields
- val newUpdates = updates.map(u => u.copy(ref = u.ref.tail))
- val updatedFieldExprs = applyUpdates(
- ArraySeq.unsafeWrapArray(fieldExprs),
- newUpdates,
- resolver,
- namePrefix :+ col.name)
-
- // construct a new struct with updated field expressions
- toNamedStruct(ArraySeq.unsafeWrapArray(fields), updatedFieldExprs)
-
- case otherType =>
- val colName = (namePrefix :+ col.name).mkString(".")
- throw new AnalysisException(
- "Updating nested fields is only supported for StructType " +
- s"but $colName is of type $otherType"
- )
- }
-
- // if there are conflicting updates, throw an exception
- // there are two illegal scenarios:
- // - multiple updates to the same column
- // - updates to a top-level struct and its nested fields (e.g., a.b and a.b.c)
- case updates if hasExactMatch(updates, col, resolver) =>
- val conflictingCols = updates.map(u => (namePrefix ++ u.ref).mkString("."))
- throw new AnalysisException(
- "Updates are in conflict for these columns: " +
- conflictingCols.distinct.mkString(", "))
- }
- }
- }
-
- private def toNamedStruct(fields: Seq[StructField], fieldExprs: Seq[Expression]): Expression = {
- val namedStructExprs = fields.zip(fieldExprs).flatMap { case (field, expr) =>
- Seq(Literal(field.name), expr)
- }
- CreateNamedStruct(namedStructExprs)
- }
-
- private def hasExactMatch(
- updates: Seq[ColumnUpdate],
- col: NamedExpression,
- resolver: Resolver): Boolean = {
-
- updates.exists(assignment => isExactMatch(assignment, col, resolver))
- }
-
- private def isExactMatch(
- update: ColumnUpdate,
- col: NamedExpression,
- resolver: Resolver): Boolean = {
-
- update.ref match {
- case Seq(namePart) if resolver(namePart, col.name) => true
- case _ => false
- }
- }
-
- protected def castIfNeeded(
- tableAttr: NamedExpression,
- expr: Expression,
- resolver: Resolver): Expression = {
-
- val storeAssignmentPolicy = conf.storeAssignmentPolicy
-
- // run the type check and catch type errors
- storeAssignmentPolicy match {
- case StoreAssignmentPolicy.STRICT | StoreAssignmentPolicy.ANSI =>
- if (expr.nullable && !tableAttr.nullable) {
- throw new AnalysisException(
- s"Cannot write nullable values to non-null column '${tableAttr.name}'")
- }
-
- // use byName = true to catch cases when struct field names don't match
- // e.g. a struct with fields (a, b) is assigned as a struct with fields (a, c) or (b, a)
- val errors = new mutable.ArrayBuffer[String]()
- val canWrite = DataType.canWrite(
- expr.dataType, tableAttr.dataType, byName = true, resolver, tableAttr.name,
- storeAssignmentPolicy, err => errors += err)
-
- if (!canWrite) {
- throw new AnalysisException(
- s"Cannot write incompatible data:\n- ${errors.mkString("\n- ")}")
- }
-
- case _ => // OK
- }
-
- storeAssignmentPolicy match {
- case _ if tableAttr.dataType.sameType(expr.dataType) =>
- expr
- case StoreAssignmentPolicy.ANSI =>
- AnsiCast(expr, tableAttr.dataType, Option(conf.sessionLocalTimeZone))
- case _ =>
- Cast(expr, tableAttr.dataType, Option(conf.sessionLocalTimeZone))
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
deleted file mode 100644
index 70f6694af6..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
-import org.apache.spark.sql.catalyst.plans.logical.InsertAction
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
-import org.apache.spark.sql.catalyst.rules.Rule
-
-/**
- * A rule that checks MERGE operations contain only supported conditions.
- *
- * Note that this rule must be run in the resolution batch before Spark executes CheckAnalysis.
- * Otherwise, CheckAnalysis will throw a less descriptive error.
- */
-object CheckMergeIntoTableConditions extends Rule[LogicalPlan] {
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case m: MergeIntoIcebergTable if m.resolved =>
- checkMergeIntoCondition("SEARCH", m.mergeCondition)
-
- val actions = m.matchedActions ++ m.notMatchedActions
- actions.foreach {
- case DeleteAction(Some(cond)) => checkMergeIntoCondition("DELETE", cond)
- case UpdateAction(Some(cond), _) => checkMergeIntoCondition("UPDATE", cond)
- case InsertAction(Some(cond), _) => checkMergeIntoCondition("INSERT", cond)
- case _ => // OK
- }
-
- m
- }
-
- private def checkMergeIntoCondition(condName: String, cond: Expression): Unit = {
- if (!cond.deterministic) {
- throw new AnalysisException(
- s"Non-deterministic functions are not supported in $condName conditions of " +
- s"MERGE operations: ${cond.sql}")
- }
-
- if (SubqueryExpression.hasSubquery(cond)) {
- throw new AnalysisException(
- s"Subqueries are not supported in conditions of MERGE operations. " +
- s"Found a subquery in the $condName condition: ${cond.sql}")
- }
-
- if (cond.find(_.isInstanceOf[AggregateExpression]).isDefined) {
- throw new AnalysisException(
- s"Agg functions are not supported in $condName conditions of MERGE operations: " + {cond.sql})
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala
deleted file mode 100644
index b3a9bda280..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable
-
-object MergeIntoIcebergTableResolutionCheck extends (LogicalPlan => Unit) {
-
- override def apply(plan: LogicalPlan): Unit = {
- plan foreach {
- case m: UnresolvedMergeIntoIcebergTable =>
- throw new AnalysisException(s"Could not resolve Iceberg MERGE INTO statement: $m")
- case _ => // OK
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
deleted file mode 100644
index 7f0ca8fadd..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.Cast
-import org.apache.spark.sql.catalyst.plans.logical.Call
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.rules.Rule
-
-object ProcedureArgumentCoercion extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case c @ Call(procedure, args) if c.resolved =>
- val params = procedure.parameters
-
- val newArgs = args.zipWithIndex.map { case (arg, index) =>
- val param = params(index)
- val paramType = param.dataType
- val argType = arg.dataType
-
- if (paramType != argType && !Cast.canUpCast(argType, paramType)) {
- throw new AnalysisException(
- s"Wrong arg type for ${param.name}: cannot cast $argType to $paramType")
- }
-
- if (paramType != argType) {
- Cast(arg, paramType)
- } else {
- arg
- }
- }
-
- if (newArgs != args) {
- c.copy(args = newArgs)
- } else {
- c
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
deleted file mode 100644
index 63ebdef957..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.plans.logical.Assignment
-import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
-import org.apache.spark.sql.catalyst.plans.logical.InsertAction
-import org.apache.spark.sql.catalyst.plans.logical.InsertStarAction
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
-import org.apache.spark.sql.catalyst.plans.logical.UpdateStarAction
-import org.apache.spark.sql.catalyst.rules.Rule
-
-/**
- * A resolution rule similar to ResolveReferences in Spark but handles Iceberg MERGE operations.
- */
-case class ResolveMergeIntoTableReferences(spark: SparkSession) extends Rule[LogicalPlan] {
-
- private lazy val analyzer: Analyzer = spark.sessionState.analyzer
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
- case m @ UnresolvedMergeIntoIcebergTable(targetTable, sourceTable, context)
- if targetTable.resolved && sourceTable.resolved && m.duplicateResolved =>
-
- val resolvedMatchedActions = context.matchedActions.map {
- case DeleteAction(cond) =>
- val resolvedCond = cond.map(resolveCond("DELETE", _, m))
- DeleteAction(resolvedCond)
-
- case UpdateAction(cond, assignments) =>
- val resolvedCond = cond.map(resolveCond("UPDATE", _, m))
- // the update action can access columns from both target and source tables
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = false)
- UpdateAction(resolvedCond, resolvedAssignments)
-
- case UpdateStarAction(updateCondition) =>
- val resolvedUpdateCondition = updateCondition.map(resolveCond("UPDATE", _, m))
- val assignments = targetTable.output.map { attr =>
- Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
- }
- // for UPDATE *, the value must be from the source table
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
- UpdateAction(resolvedUpdateCondition, resolvedAssignments)
-
- case _ =>
- throw new AnalysisException("Matched actions can only contain UPDATE or DELETE")
- }
-
- val resolvedNotMatchedActions = context.notMatchedActions.map {
- case InsertAction(cond, assignments) =>
- // the insert action is used when not matched, so its condition and value can only
- // access columns from the source table
- val resolvedCond = cond.map(resolveCond("INSERT", _, Project(Nil, m.sourceTable)))
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
- InsertAction(resolvedCond, resolvedAssignments)
-
- case InsertStarAction(cond) =>
- // the insert action is used when not matched, so its condition and value can only
- // access columns from the source table
- val resolvedCond = cond.map(resolveCond("INSERT", _, Project(Nil, m.sourceTable)))
- val assignments = targetTable.output.map { attr =>
- Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
- }
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
- InsertAction(resolvedCond, resolvedAssignments)
-
- case _ =>
- throw new AnalysisException("Not matched actions can only contain INSERT")
- }
-
- val resolvedMergeCondition = resolveCond("SEARCH", context.mergeCondition, m)
-
- MergeIntoIcebergTable(
- targetTable,
- sourceTable,
- mergeCondition = resolvedMergeCondition,
- matchedActions = resolvedMatchedActions,
- notMatchedActions = resolvedNotMatchedActions)
- }
-
- private def resolveCond(condName: String, cond: Expression, plan: LogicalPlan): Expression = {
- val resolvedCond = analyzer.resolveExpressionByPlanChildren(cond, plan)
-
- val unresolvedAttrs = resolvedCond.references.filter(!_.resolved)
- if (unresolvedAttrs.nonEmpty) {
- throw new AnalysisException(
- s"Cannot resolve ${unresolvedAttrs.map(_.sql).mkString("[", ",", "]")} in $condName condition " +
- s"of MERGE operation given input columns: ${plan.inputSet.toSeq.map(_.sql).mkString("[", ",", "]")}")
- }
-
- resolvedCond
- }
-
- // copied from ResolveReferences in Spark
- private def resolveAssignments(
- assignments: Seq[Assignment],
- mergeInto: UnresolvedMergeIntoIcebergTable,
- resolveValuesWithSourceOnly: Boolean): Seq[Assignment] = {
- assignments.map { assign =>
- val resolvedKey = assign.key match {
- case c if !c.resolved =>
- resolveMergeExprOrFail(c, Project(Nil, mergeInto.targetTable))
- case o => o
- }
- val resolvedValue = assign.value match {
- // The update values may contain target and/or source references.
- case c if !c.resolved =>
- if (resolveValuesWithSourceOnly) {
- resolveMergeExprOrFail(c, Project(Nil, mergeInto.sourceTable))
- } else {
- resolveMergeExprOrFail(c, mergeInto)
- }
- case o => o
- }
- Assignment(resolvedKey, resolvedValue)
- }
- }
-
- // copied from ResolveReferences in Spark
- private def resolveMergeExprOrFail(e: Expression, p: LogicalPlan): Expression = {
- val resolved = analyzer.resolveExpressionByPlanChildren(e, p)
- resolved.references.filter(!_.resolved).foreach { a =>
- // Note: This will throw error only on unresolved attribute issues,
- // not other resolution errors like mismatched data types.
- val cols = p.inputSet.toSeq.map(_.sql).mkString(", ")
- a.failAnalysis(s"cannot resolve ${a.sql} in MERGE command given columns [$cols]")
- }
- resolved
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
deleted file mode 100644
index ee69b5e344..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import java.util.Locale
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.plans.logical.Call
-import org.apache.spark.sql.catalyst.plans.logical.CallArgument
-import org.apache.spark.sql.catalyst.plans.logical.CallStatement
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.NamedArgument
-import org.apache.spark.sql.catalyst.plans.logical.PositionalArgument
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.connector.catalog.CatalogPlugin
-import org.apache.spark.sql.connector.catalog.LookupCatalog
-import org.apache.spark.sql.connector.iceberg.catalog.ProcedureCatalog
-import org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter
-import scala.collection.Seq
-
-case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog {
-
- protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case CallStatement(CatalogAndIdentifier(catalog, ident), args) =>
- val procedure = catalog.asProcedureCatalog.loadProcedure(ident)
-
- val params = procedure.parameters
- val normalizedParams = normalizeParams(params)
- validateParams(normalizedParams)
-
- val normalizedArgs = normalizeArgs(args)
- Call(procedure, args = buildArgExprs(normalizedParams, normalizedArgs).toSeq)
- }
-
- private def validateParams(params: Seq[ProcedureParameter]): Unit = {
- // should not be any duplicate param names
- val duplicateParamNames = params.groupBy(_.name).collect {
- case (name, matchingParams) if matchingParams.length > 1 => name
- }
-
- if (duplicateParamNames.nonEmpty) {
- throw new AnalysisException(s"Duplicate parameter names: ${duplicateParamNames.mkString("[", ",", "]")}")
- }
-
- // optional params should be at the end
- params.sliding(2).foreach {
- case Seq(previousParam, currentParam) if !previousParam.required && currentParam.required =>
- throw new AnalysisException(
- s"Optional parameters must be after required ones but $currentParam is after $previousParam")
- case _ =>
- }
- }
-
- private def buildArgExprs(
- params: Seq[ProcedureParameter],
- args: Seq[CallArgument]): Seq[Expression] = {
-
- // build a map of declared parameter names to their positions
- val nameToPositionMap = params.map(_.name).zipWithIndex.toMap
-
- // build a map of parameter names to args
- val nameToArgMap = buildNameToArgMap(params, args, nameToPositionMap)
-
- // verify all required parameters are provided
- val missingParamNames = params.filter(_.required).collect {
- case param if !nameToArgMap.contains(param.name) => param.name
- }
-
- if (missingParamNames.nonEmpty) {
- throw new AnalysisException(s"Missing required parameters: ${missingParamNames.mkString("[", ",", "]")}")
- }
-
- val argExprs = new Array[Expression](params.size)
-
- nameToArgMap.foreach { case (name, arg) =>
- val position = nameToPositionMap(name)
- argExprs(position) = arg.expr
- }
-
- // assign nulls to optional params that were not set
- params.foreach {
- case p if !p.required && !nameToArgMap.contains(p.name) =>
- val position = nameToPositionMap(p.name)
- argExprs(position) = Literal.create(null, p.dataType)
- case _ =>
- }
-
- argExprs
- }
-
- private def buildNameToArgMap(
- params: Seq[ProcedureParameter],
- args: Seq[CallArgument],
- nameToPositionMap: Map[String, Int]): Map[String, CallArgument] = {
-
- val containsNamedArg = args.exists(_.isInstanceOf[NamedArgument])
- val containsPositionalArg = args.exists(_.isInstanceOf[PositionalArgument])
-
- if (containsNamedArg && containsPositionalArg) {
- throw new AnalysisException("Named and positional arguments cannot be mixed")
- }
-
- if (containsNamedArg) {
- buildNameToArgMapUsingNames(args, nameToPositionMap)
- } else {
- buildNameToArgMapUsingPositions(args, params)
- }
- }
-
- private def buildNameToArgMapUsingNames(
- args: Seq[CallArgument],
- nameToPositionMap: Map[String, Int]): Map[String, CallArgument] = {
-
- val namedArgs = args.asInstanceOf[Seq[NamedArgument]]
-
- val validationErrors = namedArgs.groupBy(_.name).collect {
- case (name, matchingArgs) if matchingArgs.size > 1 => s"Duplicate procedure argument: $name"
- case (name, _) if !nameToPositionMap.contains(name) => s"Unknown argument: $name"
- }
-
- if (validationErrors.nonEmpty) {
- throw new AnalysisException(s"Could not build name to arg map: ${validationErrors.mkString(", ")}")
- }
-
- namedArgs.map(arg => arg.name -> arg).toMap
- }
-
- private def buildNameToArgMapUsingPositions(
- args: Seq[CallArgument],
- params: Seq[ProcedureParameter]): Map[String, CallArgument] = {
-
- if (args.size > params.size) {
- throw new AnalysisException("Too many arguments for procedure")
- }
-
- args.zipWithIndex.map { case (arg, position) =>
- val param = params(position)
- param.name -> arg
- }.toMap
- }
-
- private def normalizeParams(params: Seq[ProcedureParameter]): Seq[ProcedureParameter] = {
- params.map {
- case param if param.required =>
- val normalizedName = param.name.toLowerCase(Locale.ROOT)
- ProcedureParameter.required(normalizedName, param.dataType)
- case param =>
- val normalizedName = param.name.toLowerCase(Locale.ROOT)
- ProcedureParameter.optional(normalizedName, param.dataType)
- }
- }
-
- private def normalizeArgs(args: Seq[CallArgument]): Seq[CallArgument] = {
- args.map {
- case a @ NamedArgument(name, _) => a.copy(name = name.toLowerCase(Locale.ROOT))
- case other => other
- }
- }
-
- implicit class CatalogHelper(plugin: CatalogPlugin) {
- def asProcedureCatalog: ProcedureCatalog = plugin match {
- case procedureCatalog: ProcedureCatalog =>
- procedureCatalog
- case _ =>
- throw new AnalysisException(s"Cannot use catalog ${plugin.name}: not a ProcedureCatalog")
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromIcebergTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromIcebergTable.scala
deleted file mode 100644
index 501ef8200c..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromIcebergTable.scala
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.expressions.EqualNullSafe
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.Not
-import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
-import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
-import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations
-import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
-import org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE
-import org.apache.spark.sql.connector.write.RowLevelOperationTable
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
-
-/**
- * Assigns a rewrite plan for v2 tables that support rewriting data to handle DELETE statements.
- *
- * If a table implements SupportsDelete and SupportsRowLevelOperations, this rule assigns a rewrite
- * plan but the optimizer will check whether this particular DELETE statement can be handled
- * by simply passing delete filters to the connector. If yes, the optimizer will then discard
- * the rewrite plan.
- */
-object RewriteDeleteFromIcebergTable extends RewriteRowLevelIcebergCommand {
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case d @ DeleteFromIcebergTable(aliasedTable, Some(cond), None) if d.resolved =>
- EliminateSubqueryAliases(aliasedTable) match {
- case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) =>
- val table = buildOperationTable(tbl, DELETE, CaseInsensitiveStringMap.empty())
- val rewritePlan = table.operation match {
- case _: SupportsDelta =>
- buildWriteDeltaPlan(r, table, cond)
- case _ =>
- buildReplaceDataPlan(r, table, cond)
- }
- // keep the original relation in DELETE to try deleting using filters
- DeleteFromIcebergTable(r, Some(cond), Some(rewritePlan))
-
- case p =>
- throw new AnalysisException(s"$p is not an Iceberg table")
- }
- }
-
- // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
- private def buildReplaceDataPlan(
- relation: DataSourceV2Relation,
- operationTable: RowLevelOperationTable,
- cond: Expression): ReplaceIcebergData = {
-
- // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
- val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
-
- // construct a read relation and include all required metadata columns
- val readRelation = buildRelationWithAttrs(relation, operationTable, metadataAttrs)
-
- // construct a plan that contains unmatched rows in matched groups that must be carried over
- // such rows do not match the condition but have to be copied over as the source can replace
- // only groups of rows
- val remainingRowsFilter = Not(EqualNullSafe(cond, Literal.TrueLiteral))
- val remainingRowsPlan = Filter(remainingRowsFilter, readRelation)
-
- // build a plan to replace read groups in the table
- val writeRelation = relation.copy(table = operationTable)
- ReplaceIcebergData(writeRelation, remainingRowsPlan, relation)
- }
-
- // build a rewrite plan for sources that support row deltas
- private def buildWriteDeltaPlan(
- relation: DataSourceV2Relation,
- operationTable: RowLevelOperationTable,
- cond: Expression): WriteDelta = {
-
- // resolve all needed attrs (e.g. row ID and any required metadata attrs)
- val rowIdAttrs = resolveRowIdAttrs(relation, operationTable.operation)
- val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
-
- // construct a read relation and include all required metadata columns
- val readRelation = buildRelationWithAttrs(relation, operationTable, rowIdAttrs ++ metadataAttrs)
-
- // construct a plan that only contains records to delete
- val deletedRowsPlan = Filter(cond, readRelation)
- val operationType = Alias(Literal(DELETE_OPERATION), OPERATION_COLUMN)()
- val requiredWriteAttrs = dedupAttrs(rowIdAttrs ++ metadataAttrs)
- val project = Project(operationType +: requiredWriteAttrs, deletedRowsPlan)
-
- // build a plan to write deletes to the table
- val writeRelation = relation.copy(table = operationTable)
- val projections = buildWriteDeltaProjections(project, Nil, rowIdAttrs, metadataAttrs)
- WriteDelta(writeRelation, project, relation, projections)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
deleted file mode 100644
index c367b07c70..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.ProjectingInternalRow
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.expressions.And
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
-import org.apache.spark.sql.catalyst.expressions.IsNotNull
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
-import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
-import org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID
-import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.plans.FullOuter
-import org.apache.spark.sql.catalyst.plans.Inner
-import org.apache.spark.sql.catalyst.plans.LeftAnti
-import org.apache.spark.sql.catalyst.plans.LeftOuter
-import org.apache.spark.sql.catalyst.plans.RightOuter
-import org.apache.spark.sql.catalyst.plans.logical.AppendData
-import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
-import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.plans.logical.HintInfo
-import org.apache.spark.sql.catalyst.plans.logical.InsertAction
-import org.apache.spark.sql.catalyst.plans.logical.Join
-import org.apache.spark.sql.catalyst.plans.logical.JoinHint
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeAction
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.MergeRows
-import org.apache.spark.sql.catalyst.plans.logical.NO_BROADCAST_HASH
-import org.apache.spark.sql.catalyst.plans.logical.NoStatsUnaryNode
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
-import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
-import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
-import org.apache.spark.sql.catalyst.util.WriteDeltaProjections
-import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.NamedReference
-import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
-import org.apache.spark.sql.connector.write.RowLevelOperation.Command.MERGE
-import org.apache.spark.sql.connector.write.RowLevelOperationTable
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.types.IntegerType
-import org.apache.spark.sql.types.StructField
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
-
-/**
- * Assigns a rewrite plan for v2 tables that support rewriting data to handle MERGE statements.
- *
- * This rule assumes the commands have been fully resolved and all assignments have been aligned.
- * That's why it must be run after AlignRowLevelCommandAssignments.
- */
-object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with PredicateHelper {
-
- private final val ROW_FROM_SOURCE = "__row_from_source"
- private final val ROW_FROM_TARGET = "__row_from_target"
- private final val ROW_ID = "__row_id"
-
- private final val ROW_FROM_SOURCE_REF = FieldReference(ROW_FROM_SOURCE)
- private final val ROW_FROM_TARGET_REF = FieldReference(ROW_FROM_TARGET)
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
- if m.resolved && m.aligned && matchedActions.isEmpty && notMatchedActions.size == 1 =>
-
- EliminateSubqueryAliases(aliasedTable) match {
- case r: DataSourceV2Relation =>
- // NOT MATCHED conditions may only refer to columns in source so they can be pushed down
- val insertAction = notMatchedActions.head.asInstanceOf[InsertAction]
- val filteredSource = insertAction.condition match {
- case Some(insertCond) => Filter(insertCond, source)
- case None => source
- }
-
- // when there are no MATCHED actions, use a left anti join to remove any matching rows
- // and switch to using a regular append instead of a row-level merge
- // only unmatched source rows that match the condition are appended to the table
- val joinPlan = Join(filteredSource, r, LeftAnti, Some(cond), JoinHint.NONE)
-
- val outputExprs = insertAction.assignments.map(_.value)
- val outputColNames = r.output.map(_.name)
- val outputCols = outputExprs.zip(outputColNames).map { case (expr, name) =>
- Alias(expr, name)()
- }
- val project = Project(outputCols, joinPlan)
-
- AppendData.byPosition(r, project)
-
- case p =>
- throw new AnalysisException(s"$p is not an Iceberg table")
- }
-
- case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
- if m.resolved && m.aligned && matchedActions.isEmpty =>
-
- EliminateSubqueryAliases(aliasedTable) match {
- case r: DataSourceV2Relation =>
- // when there are no MATCHED actions, use a left anti join to remove any matching rows
- // and switch to using a regular append instead of a row-level merge
- // only unmatched source rows that match action conditions are appended to the table
- val joinPlan = Join(source, r, LeftAnti, Some(cond), JoinHint.NONE)
-
- val notMatchedConditions = notMatchedActions.map(actionCondition)
- val notMatchedOutputs = notMatchedActions.map(actionOutput(_, Nil))
-
- // merge rows as there are multiple not matched actions
- val mergeRows = MergeRows(
- isSourceRowPresent = TrueLiteral,
- isTargetRowPresent = FalseLiteral,
- matchedConditions = Nil,
- matchedOutputs = Nil,
- notMatchedConditions = notMatchedConditions,
- notMatchedOutputs = notMatchedOutputs,
- targetOutput = Nil,
- performCardinalityCheck = false,
- emitNotMatchedTargetRows = false,
- output = buildMergeRowsOutput(Nil, notMatchedOutputs, r.output),
- joinPlan)
-
- AppendData.byPosition(r, mergeRows)
-
- case p =>
- throw new AnalysisException(s"$p is not an Iceberg table")
- }
-
- case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
- if m.resolved && m.aligned =>
-
- EliminateSubqueryAliases(aliasedTable) match {
- case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) =>
- val table = buildOperationTable(tbl, MERGE, CaseInsensitiveStringMap.empty())
- val rewritePlan = table.operation match {
- case _: SupportsDelta =>
- buildWriteDeltaPlan(r, table, source, cond, matchedActions, notMatchedActions)
- case _ =>
- buildReplaceDataPlan(r, table, source, cond, matchedActions, notMatchedActions)
- }
-
- m.copy(rewritePlan = Some(rewritePlan))
-
- case p =>
- throw new AnalysisException(s"$p is not an Iceberg table")
- }
- }
-
- // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
- private def buildReplaceDataPlan(
- relation: DataSourceV2Relation,
- operationTable: RowLevelOperationTable,
- source: LogicalPlan,
- cond: Expression,
- matchedActions: Seq[MergeAction],
- notMatchedActions: Seq[MergeAction]): ReplaceIcebergData = {
-
- // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
- val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
-
- // construct a scan relation and include all required metadata columns
- val readRelation = buildRelationWithAttrs(relation, operationTable, metadataAttrs)
- val readAttrs = readRelation.output
-
- val performCardinalityCheck = isCardinalityCheckNeeded(matchedActions)
-
- // project an extra column to check if a target row exists after the join
- // if needed, project a synthetic row ID to perform the cardinality check
- val rowFromTarget = Alias(TrueLiteral, ROW_FROM_TARGET)()
- val targetTableProjExprs = if (performCardinalityCheck) {
- val rowId = Alias(MonotonicallyIncreasingID(), ROW_ID)()
- readAttrs ++ Seq(rowFromTarget, rowId)
- } else {
- readAttrs :+ rowFromTarget
- }
- val targetTableProj = Project(targetTableProjExprs, readRelation)
-
- // project an extra column to check if a source row exists after the join
- val rowFromSource = Alias(TrueLiteral, ROW_FROM_SOURCE)()
- val sourceTableProjExprs = source.output :+ rowFromSource
- val sourceTableProj = Project(sourceTableProjExprs, source)
-
- // use left outer join if there is no NOT MATCHED action, unmatched source rows can be discarded
- // use full outer join in all other cases, unmatched source rows may be needed
- // disable broadcasts for the target table to perform the cardinality check
- val joinType = if (notMatchedActions.isEmpty) LeftOuter else FullOuter
- val joinHint = JoinHint(leftHint = Some(HintInfo(Some(NO_BROADCAST_HASH))), rightHint = None)
- val joinPlan = Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(cond), joinHint)
-
- // add an extra matched action to output the original row if none of the actual actions matched
- // this is needed to keep target rows that should be copied over
- val matchedConditions = matchedActions.map(actionCondition) :+ TrueLiteral
- val matchedOutputs = matchedActions.map(actionOutput(_, metadataAttrs)) :+ readAttrs
-
- val notMatchedConditions = notMatchedActions.map(actionCondition)
- val notMatchedOutputs = notMatchedActions.map(actionOutput(_, metadataAttrs))
-
- val rowFromSourceAttr = resolveAttrRef(ROW_FROM_SOURCE_REF, joinPlan)
- val rowFromTargetAttr = resolveAttrRef(ROW_FROM_TARGET_REF, joinPlan)
-
- val mergeRows = MergeRows(
- isSourceRowPresent = IsNotNull(rowFromSourceAttr),
- isTargetRowPresent = if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
- matchedConditions = matchedConditions,
- matchedOutputs = matchedOutputs,
- notMatchedConditions = notMatchedConditions,
- notMatchedOutputs = notMatchedOutputs,
- targetOutput = readAttrs,
- performCardinalityCheck = performCardinalityCheck,
- emitNotMatchedTargetRows = true,
- output = buildMergeRowsOutput(matchedOutputs, notMatchedOutputs, readAttrs),
- joinPlan)
-
- // build a plan to replace read groups in the table
- val writeRelation = relation.copy(table = operationTable)
- ReplaceIcebergData(writeRelation, mergeRows, relation)
- }
-
- // build a rewrite plan for sources that support row deltas
- private def buildWriteDeltaPlan(
- relation: DataSourceV2Relation,
- operationTable: RowLevelOperationTable,
- source: LogicalPlan,
- cond: Expression,
- matchedActions: Seq[MergeAction],
- notMatchedActions: Seq[MergeAction]): WriteDelta = {
-
- // resolve all needed attrs (e.g. row ID and any required metadata attrs)
- val rowAttrs = relation.output
- val rowIdAttrs = resolveRowIdAttrs(relation, operationTable.operation)
- val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
-
- // construct a scan relation and include all required metadata columns
- val readRelation = buildRelationWithAttrs(relation, operationTable, rowIdAttrs ++ metadataAttrs)
- val readAttrs = readRelation.output
-
- val (targetCond, joinCond) = splitMergeCond(cond, readRelation)
-
- val performCardinalityCheck = isCardinalityCheckNeeded(matchedActions)
-
- // project an extra column to check if a target row exists after the join
- // if needed, project a synthetic row ID to perform the cardinality check
- val rowFromTarget = Alias(TrueLiteral, ROW_FROM_TARGET)()
- val targetTableProjExprs = if (performCardinalityCheck) {
- val rowId = Alias(MonotonicallyIncreasingID(), ROW_ID)()
- readAttrs ++ Seq(rowFromTarget, rowId)
- } else {
- readAttrs :+ rowFromTarget
- }
- val targetTableProj = Project(targetTableProjExprs, Filter(targetCond, readRelation))
-
- // project an extra column to check if a source row exists after the join
- val sourceTableProjExprs = source.output :+ Alias(TrueLiteral, ROW_FROM_SOURCE)()
- val sourceTableProj = Project(sourceTableProjExprs, source)
-
- // use inner join if there is no NOT MATCHED action, unmatched source rows can be discarded
- // use right outer join in all other cases, unmatched source rows may be needed
- // also disable broadcasts for the target table to perform the cardinality check
- val joinType = if (notMatchedActions.isEmpty) Inner else RightOuter
- val joinHint = JoinHint(leftHint = Some(HintInfo(Some(NO_BROADCAST_HASH))), rightHint = None)
- val joinPlan = Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(joinCond), joinHint)
-
- val deleteRowValues = buildDeltaDeleteRowValues(rowAttrs, rowIdAttrs)
- val metadataReadAttrs = readAttrs.filterNot(relation.outputSet.contains)
-
- val matchedConditions = matchedActions.map(actionCondition)
- val matchedOutputs = matchedActions.map(deltaActionOutput(_, deleteRowValues, metadataReadAttrs))
-
- val notMatchedConditions = notMatchedActions.map(actionCondition)
- val notMatchedOutputs = notMatchedActions.map(deltaActionOutput(_, deleteRowValues, metadataReadAttrs))
-
- val operationTypeAttr = AttributeReference(OPERATION_COLUMN, IntegerType, nullable = false)()
- val rowFromSourceAttr = resolveAttrRef(ROW_FROM_SOURCE_REF, joinPlan)
- val rowFromTargetAttr = resolveAttrRef(ROW_FROM_TARGET_REF, joinPlan)
-
- // merged rows must contain values for the operation type and all read attrs
- val mergeRowsOutput = buildMergeRowsOutput(matchedOutputs, notMatchedOutputs, operationTypeAttr +: readAttrs)
-
- val mergeRows = MergeRows(
- isSourceRowPresent = IsNotNull(rowFromSourceAttr),
- isTargetRowPresent = if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
- matchedConditions = matchedConditions,
- matchedOutputs = matchedOutputs,
- notMatchedConditions = notMatchedConditions,
- notMatchedOutputs = notMatchedOutputs,
- // only needed if emitting unmatched target rows
- targetOutput = Nil,
- performCardinalityCheck = performCardinalityCheck,
- emitNotMatchedTargetRows = false,
- output = mergeRowsOutput,
- joinPlan)
-
- // build a plan to write the row delta to the table
- val writeRelation = relation.copy(table = operationTable)
- val projections = buildMergeDeltaProjections(mergeRows, rowAttrs, rowIdAttrs, metadataAttrs)
- WriteDelta(writeRelation, mergeRows, relation, projections)
- }
-
- private def actionCondition(action: MergeAction): Expression = {
- action.condition.getOrElse(TrueLiteral)
- }
-
- private def actionOutput(
- clause: MergeAction,
- metadataAttrs: Seq[Attribute]): Seq[Expression] = {
-
- clause match {
- case u: UpdateAction =>
- u.assignments.map(_.value) ++ metadataAttrs
-
- case _: DeleteAction =>
- Nil
-
- case i: InsertAction =>
- i.assignments.map(_.value) ++ metadataAttrs.map(attr => Literal(null, attr.dataType))
-
- case other =>
- throw new AnalysisException(s"Unexpected action: $other")
- }
- }
-
- private def deltaActionOutput(
- action: MergeAction,
- deleteRowValues: Seq[Expression],
- metadataAttrs: Seq[Attribute]): Seq[Expression] = {
-
- action match {
- case u: UpdateAction =>
- Seq(Literal(UPDATE_OPERATION)) ++ u.assignments.map(_.value) ++ metadataAttrs
-
- case _: DeleteAction =>
- Seq(Literal(DELETE_OPERATION)) ++ deleteRowValues ++ metadataAttrs
-
- case i: InsertAction =>
- val metadataAttrValues = metadataAttrs.map(attr => Literal(null, attr.dataType))
- Seq(Literal(INSERT_OPERATION)) ++ i.assignments.map(_.value) ++ metadataAttrValues
-
- case other =>
- throw new AnalysisException(s"Unexpected action: $other")
- }
- }
-
- private def buildMergeRowsOutput(
- matchedOutputs: Seq[Seq[Expression]],
- notMatchedOutputs: Seq[Seq[Expression]],
- attrs: Seq[Attribute]): Seq[Attribute] = {
-
- // collect all outputs from matched and not matched actions (ignoring DELETEs)
- val outputs = matchedOutputs.filter(_.nonEmpty) ++ notMatchedOutputs.filter(_.nonEmpty)
-
- // build a correct nullability map for output attributes
- // an attribute is nullable if at least one matched or not matched action may produce null
- val nullabilityMap = attrs.indices.map { index =>
- index -> outputs.exists(output => output(index).nullable)
- }.toMap
-
- attrs.zipWithIndex.map { case (attr, index) =>
- AttributeReference(attr.name, attr.dataType, nullabilityMap(index), attr.metadata)()
- }
- }
-
- private def isCardinalityCheckNeeded(actions: Seq[MergeAction]): Boolean = actions match {
- case Seq(DeleteAction(None)) => false
- case _ => true
- }
-
- private def buildDeltaDeleteRowValues(
- rowAttrs: Seq[Attribute],
- rowIdAttrs: Seq[Attribute]): Seq[Expression] = {
-
- // nullify all row attrs that are not part of the row ID
- val rowIdAttSet = AttributeSet(rowIdAttrs)
- rowAttrs.map {
- case attr if rowIdAttSet.contains(attr) => attr
- case attr => Literal(null, attr.dataType)
- }
- }
-
- private def resolveAttrRef(ref: NamedReference, plan: LogicalPlan): AttributeReference = {
- ExtendedV2ExpressionUtils.resolveRef[AttributeReference](ref, plan)
- }
-
- private def buildMergeDeltaProjections(
- mergeRows: MergeRows,
- rowAttrs: Seq[Attribute],
- rowIdAttrs: Seq[Attribute],
- metadataAttrs: Seq[Attribute]): WriteDeltaProjections = {
-
- val outputAttrs = mergeRows.output
-
- val outputs = mergeRows.matchedOutputs ++ mergeRows.notMatchedOutputs
- val insertAndUpdateOutputs = outputs.filterNot(_.head == Literal(DELETE_OPERATION))
- val updateAndDeleteOutputs = outputs.filterNot(_.head == Literal(INSERT_OPERATION))
-
- val rowProjection = if (rowAttrs.nonEmpty) {
- Some(newLazyProjection(insertAndUpdateOutputs, outputAttrs, rowAttrs))
- } else {
- None
- }
-
- val rowIdProjection = newLazyProjection(updateAndDeleteOutputs, outputAttrs, rowIdAttrs)
-
- val metadataProjection = if (metadataAttrs.nonEmpty) {
- Some(newLazyProjection(updateAndDeleteOutputs, outputAttrs, metadataAttrs))
- } else {
- None
- }
-
- WriteDeltaProjections(rowProjection, rowIdProjection, metadataProjection)
- }
-
- // the projection is done by name, ignoring expr IDs
- private def newLazyProjection(
- outputs: Seq[Seq[Expression]],
- outputAttrs: Seq[Attribute],
- projectedAttrs: Seq[Attribute]): ProjectingInternalRow = {
-
- val projectedOrdinals = projectedAttrs.map(attr => outputAttrs.indexWhere(_.name == attr.name))
-
- val structFields = projectedAttrs.zip(projectedOrdinals).map { case (attr, ordinal) =>
- // output attr is nullable if at least one action may produce null for that attr
- // but row ID and metadata attrs are projected only in update/delete actions and
- // row attrs are projected only in insert/update actions
- // that's why the projection schema must rely only on relevant action outputs
- // instead of blindly inheriting the output attr nullability
- val nullable = outputs.exists(output => output(ordinal).nullable)
- StructField(attr.name, attr.dataType, nullable, attr.metadata)
- }
- val schema = StructType(structFields)
-
- ProjectingInternalRow(schema, projectedOrdinals)
- }
-
- // splits the MERGE condition into a predicate that references columns only from the target table,
- // which can be pushed down, and a predicate used as a join condition to find matches
- private def splitMergeCond(
- cond: Expression,
- targetTable: LogicalPlan): (Expression, Expression) = {
-
- val (targetPredicates, joinPredicates) = splitConjunctivePredicates(cond)
- .partition(_.references.subsetOf(targetTable.outputSet))
- val targetCond = targetPredicates.reduceOption(And).getOrElse(TrueLiteral)
- val joinCond = joinPredicates.reduceOption(And).getOrElse(TrueLiteral)
- (targetCond, joinCond)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelIcebergCommand.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelIcebergCommand.scala
deleted file mode 100644
index c7378f71e6..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelIcebergCommand.scala
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.ProjectingInternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.util.WriteDeltaProjections
-import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
-import org.apache.spark.sql.connector.write.RowLevelOperation
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.types.StructType
-
-trait RewriteRowLevelIcebergCommand extends RewriteRowLevelCommand {
-
- protected def buildWriteDeltaProjections(
- plan: LogicalPlan,
- rowAttrs: Seq[Attribute],
- rowIdAttrs: Seq[Attribute],
- metadataAttrs: Seq[Attribute]): WriteDeltaProjections = {
-
- val rowProjection = if (rowAttrs.nonEmpty) {
- Some(newLazyProjection(plan, rowAttrs))
- } else {
- None
- }
-
- val rowIdProjection = newLazyProjection(plan, rowIdAttrs)
-
- val metadataProjection = if (metadataAttrs.nonEmpty) {
- Some(newLazyProjection(plan, metadataAttrs))
- } else {
- None
- }
-
- WriteDeltaProjections(rowProjection, rowIdProjection, metadataProjection)
- }
-
- // the projection is done by name, ignoring expr IDs
- private def newLazyProjection(
- plan: LogicalPlan,
- projectedAttrs: Seq[Attribute]): ProjectingInternalRow = {
-
- val projectedOrdinals = projectedAttrs.map(attr => plan.output.indexWhere(_.name == attr.name))
- val schema = StructType.fromAttributes(projectedOrdinals.map(plan.output(_)))
- ProjectingInternalRow(schema, projectedOrdinals)
- }
-
- protected def resolveRowIdAttrs(
- relation: DataSourceV2Relation,
- operation: RowLevelOperation): Seq[AttributeReference] = {
-
- operation match {
- case supportsDelta: SupportsDelta =>
- val rowIdAttrs = ExtendedV2ExpressionUtils.resolveRefs[AttributeReference](
- supportsDelta.rowId.toSeq,
- relation)
-
- val nullableRowIdAttrs = rowIdAttrs.filter(_.nullable)
- if (nullableRowIdAttrs.nonEmpty) {
- throw new AnalysisException(s"Row ID attrs cannot be nullable: $nullableRowIdAttrs")
- }
-
- rowIdAttrs
-
- case other =>
- throw new AnalysisException(s"Operation $other does not support deltas")
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala
deleted file mode 100644
index f1c10c07da..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.expressions.EqualNullSafe
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.If
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.Not
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.plans.logical.Assignment
-import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.Union
-import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
-import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
-import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations
-import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
-import org.apache.spark.sql.connector.write.RowLevelOperation.Command.UPDATE
-import org.apache.spark.sql.connector.write.RowLevelOperationTable
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
-
-/**
- * Assigns a rewrite plan for v2 tables that support rewriting data to handle UPDATE statements.
- *
- * This rule assumes the commands have been fully resolved and all assignments have been aligned.
- * That's why it must be run after AlignRowLevelCommandAssignments.
- *
- * This rule also must be run in the same batch with DeduplicateRelations in Spark.
- */
-object RewriteUpdateTable extends RewriteRowLevelIcebergCommand {
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case u @ UpdateIcebergTable(aliasedTable, assignments, cond, None) if u.resolved && u.aligned =>
- EliminateSubqueryAliases(aliasedTable) match {
- case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) =>
- val table = buildOperationTable(tbl, UPDATE, CaseInsensitiveStringMap.empty())
- val updateCond = cond.getOrElse(Literal.TrueLiteral)
- val rewritePlan = table.operation match {
- case _: SupportsDelta =>
- buildWriteDeltaPlan(r, table, assignments, updateCond)
- case _ if SubqueryExpression.hasSubquery(updateCond) =>
- buildReplaceDataWithUnionPlan(r, table, assignments, updateCond)
- case _ =>
- buildReplaceDataPlan(r, table, assignments, updateCond)
- }
- UpdateIcebergTable(r, assignments, cond, Some(rewritePlan))
-
- case p =>
- throw new AnalysisException(s"$p is not an Iceberg table")
- }
- }
-
- // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
- // if the condition does NOT contain a subquery
- private def buildReplaceDataPlan(
- relation: DataSourceV2Relation,
- operationTable: RowLevelOperationTable,
- assignments: Seq[Assignment],
- cond: Expression): ReplaceIcebergData = {
-
- // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
- val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
-
- // construct a read relation and include all required metadata columns
- val readRelation = buildRelationWithAttrs(relation, operationTable, metadataAttrs)
-
- // build a plan with updated and copied over records
- val updatedAndRemainingRowsPlan = buildUpdateProjection(readRelation, assignments, cond)
-
- // build a plan to replace read groups in the table
- val writeRelation = relation.copy(table = operationTable)
- ReplaceIcebergData(writeRelation, updatedAndRemainingRowsPlan, relation)
- }
-
- // build a rewrite plan for sources that support replacing groups of data (e.g. files, partitions)
- // if the condition contains a subquery
- private def buildReplaceDataWithUnionPlan(
- relation: DataSourceV2Relation,
- operationTable: RowLevelOperationTable,
- assignments: Seq[Assignment],
- cond: Expression): ReplaceIcebergData = {
-
- // resolve all needed attrs (e.g. metadata attrs for grouping data on write)
- val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
-
- // construct a read relation and include all required metadata columns
- // the same read relation will be used to read records that must be updated and be copied over
- // DeduplicateRelations will take care of duplicated attr IDs
- val readRelation = buildRelationWithAttrs(relation, operationTable, metadataAttrs)
-
- // build a plan for records that match the cond and should be updated
- val matchedRowsPlan = Filter(cond, readRelation)
- val updatedRowsPlan = buildUpdateProjection(matchedRowsPlan, assignments)
-
- // build a plan for records that did not match the cond but had to be copied over
- val remainingRowFilter = Not(EqualNullSafe(cond, Literal.TrueLiteral))
- val remainingRowsPlan = Filter(remainingRowFilter, readRelation)
-
- // new state is a union of updated and copied over records
- val updatedAndRemainingRowsPlan = Union(updatedRowsPlan, remainingRowsPlan)
-
- // build a plan to replace read groups in the table
- val writeRelation = relation.copy(table = operationTable)
- ReplaceIcebergData(writeRelation, updatedAndRemainingRowsPlan, relation)
- }
-
- // build a rewrite plan for sources that support row deltas
- private def buildWriteDeltaPlan(
- relation: DataSourceV2Relation,
- operationTable: RowLevelOperationTable,
- assignments: Seq[Assignment],
- cond: Expression): WriteDelta = {
-
- // resolve all needed attrs (e.g. row ID and any required metadata attrs)
- val rowAttrs = relation.output
- val rowIdAttrs = resolveRowIdAttrs(relation, operationTable.operation)
- val metadataAttrs = resolveRequiredMetadataAttrs(relation, operationTable.operation)
-
- // construct a scan relation and include all required metadata columns
- val readRelation = buildRelationWithAttrs(relation, operationTable, rowIdAttrs ++ metadataAttrs)
-
- // build a plan for updated records that match the cond
- val matchedRowsPlan = Filter(cond, readRelation)
- val updatedRowsPlan = buildUpdateProjection(matchedRowsPlan, assignments)
- val operationType = Alias(Literal(UPDATE_OPERATION), OPERATION_COLUMN)()
- val project = Project(operationType +: updatedRowsPlan.output, updatedRowsPlan)
-
- // build a plan to write the row delta to the table
- val writeRelation = relation.copy(table = operationTable)
- val projections = buildWriteDeltaProjections(project, rowAttrs, rowIdAttrs, metadataAttrs)
- WriteDelta(writeRelation, project, relation, projections)
- }
-
- // this method assumes the assignments have been already aligned before
- // the condition passed to this method may be different from the UPDATE condition
- private def buildUpdateProjection(
- plan: LogicalPlan,
- assignments: Seq[Assignment],
- cond: Expression = Literal.TrueLiteral): LogicalPlan = {
-
- // TODO: avoid executing the condition for each column
-
- // the plan output may include metadata columns that are not modified
- // that's why the number of assignments may not match the number of plan output columns
-
- val assignedValues = assignments.map(_.value)
- val updatedValues = plan.output.zipWithIndex.map { case (attr, index) =>
- if (index < assignments.size) {
- val assignedExpr = assignedValues(index)
- val updatedValue = If(cond, assignedExpr, attr)
- Alias(updatedValue, attr.name)()
- } else {
- attr
- }
- }
-
- Project(updatedValues, plan)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala
deleted file mode 100644
index ce3818922c..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.expressions
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.SQLConfHelper
-import org.apache.spark.sql.catalyst.plans.logical.Assignment
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.util.CharVarcharUtils
-import org.apache.spark.sql.types.DataType
-
-object AssignmentUtils extends SQLConfHelper {
-
- /**
- * Checks whether assignments are aligned and match table columns.
- *
- * @param table a target table
- * @param assignments assignments to check
- * @return true if the assignments are aligned
- */
- def aligned(table: LogicalPlan, assignments: Seq[Assignment]): Boolean = {
- val sameSize = table.output.size == assignments.size
- sameSize && table.output.zip(assignments).forall { case (attr, assignment) =>
- val key = assignment.key
- val value = assignment.value
- val refsEqual = toAssignmentRef(attr).zip(toAssignmentRef(key))
- .forall{ case (attrRef, keyRef) => conf.resolver(attrRef, keyRef)}
-
- refsEqual &&
- DataType.equalsIgnoreCompatibleNullability(value.dataType, attr.dataType) &&
- (attr.nullable || !value.nullable)
- }
- }
-
- def toAssignmentRef(expr: Expression): Seq[String] = expr match {
- case attr: AttributeReference =>
- Seq(attr.name)
- case Alias(child, _) =>
- toAssignmentRef(child)
- case GetStructField(child, _, Some(name)) =>
- toAssignmentRef(child) :+ name
- case other: ExtractValue =>
- throw new AnalysisException(s"Updating nested fields is only supported for structs: $other")
- case other =>
- throw new AnalysisException(s"Cannot convert to a reference, unsupported expression: $other")
- }
-
- def handleCharVarcharLimits(assignment: Assignment): Assignment = {
- val key = assignment.key
- val value = assignment.value
-
- val rawKeyType = key.transform {
- case attr: AttributeReference =>
- CharVarcharUtils.getRawType(attr.metadata)
- .map(attr.withDataType)
- .getOrElse(attr)
- }.dataType
-
- if (CharVarcharUtils.hasCharVarchar(rawKeyType)) {
- val newKey = key.transform {
- case attr: AttributeReference => CharVarcharUtils.cleanAttrMetadata(attr)
- }
- val newValue = CharVarcharUtils.stringLengthCheck(value, rawKeyType)
- Assignment(newKey, newValue)
- } else {
- assignment
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtendedV2ExpressionUtils.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtendedV2ExpressionUtils.scala
deleted file mode 100644
index 16ff67a705..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtendedV2ExpressionUtils.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.expressions
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.SQLConfHelper
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.connector.expressions.{Expression => V2Expression}
-import org.apache.spark.sql.connector.expressions.{SortDirection => V2SortDirection}
-import org.apache.spark.sql.connector.expressions.{NullOrdering => V2NullOrdering}
-import org.apache.spark.sql.connector.expressions.BucketTransform
-import org.apache.spark.sql.connector.expressions.DaysTransform
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.HoursTransform
-import org.apache.spark.sql.connector.expressions.IdentityTransform
-import org.apache.spark.sql.connector.expressions.MonthsTransform
-import org.apache.spark.sql.connector.expressions.NamedReference
-import org.apache.spark.sql.connector.expressions.SortValue
-import org.apache.spark.sql.connector.expressions.Transform
-import org.apache.spark.sql.connector.expressions.TruncateTransform
-import org.apache.spark.sql.connector.expressions.YearsTransform
-import org.apache.spark.sql.errors.QueryCompilationErrors
-
-/**
- * A class that is inspired by V2ExpressionUtils in Spark but supports Iceberg transforms.
- */
-object ExtendedV2ExpressionUtils extends SQLConfHelper {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
-
- def resolveRef[T <: NamedExpression](ref: NamedReference, plan: LogicalPlan): T = {
- plan.resolve(ref.fieldNames.toSeq, conf.resolver) match {
- case Some(namedExpr) =>
- namedExpr.asInstanceOf[T]
- case None =>
- val name = ref.fieldNames.toSeq.quoted
- val outputString = plan.output.map(_.name).mkString(",")
- throw QueryCompilationErrors.cannotResolveAttributeError(name, outputString)
- }
- }
-
- def resolveRefs[T <: NamedExpression](refs: Seq[NamedReference], plan: LogicalPlan): Seq[T] = {
- refs.map(ref => resolveRef[T](ref, plan))
- }
-
- def toCatalyst(expr: V2Expression, query: LogicalPlan): Expression = {
- expr match {
- case SortValue(child, direction, nullOrdering) =>
- val catalystChild = toCatalyst(child, query)
- SortOrder(catalystChild, toCatalyst(direction), toCatalyst(nullOrdering), Seq.empty)
- case IdentityTransform(ref) =>
- resolveRef[NamedExpression](ref, query)
- case t: Transform if BucketTransform.unapply(t).isDefined =>
- t match {
- // sort columns will be empty for bucket.
- case BucketTransform(numBuckets, cols, _) =>
- IcebergBucketTransform(numBuckets, resolveRef[NamedExpression](cols.head, query))
- case _ => t.asInstanceOf[Expression]
- // do nothing
- }
- case TruncateTransform(length, ref) =>
- IcebergTruncateTransform(resolveRef[NamedExpression](ref, query), length)
- case YearsTransform(ref) =>
- IcebergYearTransform(resolveRef[NamedExpression](ref, query))
- case MonthsTransform(ref) =>
- IcebergMonthTransform(resolveRef[NamedExpression](ref, query))
- case DaysTransform(ref) =>
- IcebergDayTransform(resolveRef[NamedExpression](ref, query))
- case HoursTransform(ref) =>
- IcebergHourTransform(resolveRef[NamedExpression](ref, query))
- case ref: FieldReference =>
- resolveRef[NamedExpression](ref, query)
- case _ =>
- throw new AnalysisException(s"$expr is not currently supported")
- }
- }
-
- private def toCatalyst(direction: V2SortDirection): SortDirection = direction match {
- case V2SortDirection.ASCENDING => Ascending
- case V2SortDirection.DESCENDING => Descending
- }
-
- private def toCatalyst(nullOrdering: V2NullOrdering): NullOrdering = nullOrdering match {
- case V2NullOrdering.NULLS_FIRST => NullsFirst
- case V2NullOrdering.NULLS_LAST => NullsLast
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
deleted file mode 100644
index d62cf6d839..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.optimizer
-
-import org.apache.spark.sql.catalyst.expressions.And
-import org.apache.spark.sql.catalyst.expressions.CaseWhen
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.If
-import org.apache.spark.sql.catalyst.expressions.In
-import org.apache.spark.sql.catalyst.expressions.InSet
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
-import org.apache.spark.sql.catalyst.expressions.Not
-import org.apache.spark.sql.catalyst.expressions.Or
-import org.apache.spark.sql.catalyst.plans.logical.DeleteAction
-import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.InsertAction
-import org.apache.spark.sql.catalyst.plans.logical.InsertStarAction
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeAction
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateAction
-import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateStarAction
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.INSET
-import org.apache.spark.sql.catalyst.trees.TreePattern.NULL_LITERAL
-import org.apache.spark.sql.catalyst.trees.TreePattern.TRUE_OR_FALSE_LITERAL
-import org.apache.spark.sql.types.BooleanType
-import org.apache.spark.util.Utils
-
-/**
- * A rule similar to ReplaceNullWithFalseInPredicate in Spark but applies to Iceberg row-level commands.
- */
-object ExtendedReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
- _.containsAnyPattern(NULL_LITERAL, TRUE_OR_FALSE_LITERAL, INSET)) {
-
- case d @ DeleteFromIcebergTable(_, Some(cond), _) =>
- d.copy(condition = Some(replaceNullWithFalse(cond)))
-
- case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
- u.copy(condition = Some(replaceNullWithFalse(cond)))
-
- case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
- m.copy(
- mergeCondition = replaceNullWithFalse(mergeCond),
- matchedActions = replaceNullWithFalse(matchedActions),
- notMatchedActions = replaceNullWithFalse(notMatchedActions))
- }
-
- /**
- * Recursively traverse the Boolean-type expression to replace
- * `Literal(null, BooleanType)` with `FalseLiteral`, if possible.
- *
- * Note that `transformExpressionsDown` can not be used here as we must stop as soon as we hit
- * an expression that is not [[CaseWhen]], [[If]], [[And]], [[Or]] or
- * `Literal(null, BooleanType)`.
- */
- private def replaceNullWithFalse(e: Expression): Expression = e match {
- case Literal(null, BooleanType) =>
- FalseLiteral
- // In SQL, the `Not(IN)` expression evaluates as follows:
- // `NULL not in (1)` -> NULL
- // `NULL not in (1, NULL)` -> NULL
- // `1 not in (1, NULL)` -> false
- // `1 not in (2, NULL)` -> NULL
- // In predicate, NULL is equal to false, so we can simplify them to false directly.
- case Not(In(value, list)) if (value +: list).exists(isNullLiteral) =>
- FalseLiteral
- case Not(InSet(value, list)) if isNullLiteral(value) || list.contains(null) =>
- FalseLiteral
-
- case And(left, right) =>
- And(replaceNullWithFalse(left), replaceNullWithFalse(right))
- case Or(left, right) =>
- Or(replaceNullWithFalse(left), replaceNullWithFalse(right))
- case cw: CaseWhen if cw.dataType == BooleanType =>
- val newBranches = cw.branches.map { case (cond, value) =>
- replaceNullWithFalse(cond) -> replaceNullWithFalse(value)
- }
- val newElseValue = cw.elseValue.map(replaceNullWithFalse).getOrElse(FalseLiteral)
- CaseWhen(newBranches, newElseValue)
- case i @ If(pred, trueVal, falseVal) if i.dataType == BooleanType =>
- If(replaceNullWithFalse(pred), replaceNullWithFalse(trueVal), replaceNullWithFalse(falseVal))
- case e if e.dataType == BooleanType =>
- e
- case e =>
- val message = "Expected a Boolean type expression in replaceNullWithFalse, " +
- s"but got the type `${e.dataType.catalogString}` in `${e.sql}`."
- if (Utils.isTesting) {
- throw new IllegalArgumentException(message)
- } else {
- logWarning(message)
- e
- }
- }
-
- private def isNullLiteral(e: Expression): Boolean = e match {
- case Literal(null, _) => true
- case _ => false
- }
-
- private def replaceNullWithFalse(mergeActions: Seq[MergeAction]): Seq[MergeAction] = {
- mergeActions.map {
- case u @ UpdateAction(Some(cond), _) => u.copy(condition = Some(replaceNullWithFalse(cond)))
- case u @ UpdateStarAction(Some(cond)) => u.copy(condition = Some(replaceNullWithFalse(cond)))
- case d @ DeleteAction(Some(cond)) => d.copy(condition = Some(replaceNullWithFalse(cond)))
- case i @ InsertAction(Some(cond), _) => i.copy(condition = Some(replaceNullWithFalse(cond)))
- case i @ InsertStarAction(Some(cond)) => i.copy(condition = Some(replaceNullWithFalse(cond)))
- case other => other
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
deleted file mode 100644
index f4df565c44..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.optimizer
-
-import org.apache.spark.sql.catalyst.expressions.And
-import org.apache.spark.sql.catalyst.expressions.CaseWhen
-import org.apache.spark.sql.catalyst.expressions.Coalesce
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.If
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
-import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
-import org.apache.spark.sql.catalyst.expressions.Not
-import org.apache.spark.sql.catalyst.expressions.Or
-import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.CASE_WHEN
-import org.apache.spark.sql.catalyst.trees.TreePattern.IF
-import org.apache.spark.sql.types.BooleanType
-
-/**
- * A rule similar to SimplifyConditionalsInPredicate in Spark but applies to Iceberg row-level commands.
- */
-object ExtendedSimplifyConditionalsInPredicate extends Rule[LogicalPlan] {
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
- _.containsAnyPattern(CASE_WHEN, IF)) {
-
- case d @ DeleteFromIcebergTable(_, Some(cond), _) =>
- d.copy(condition = Some(simplifyConditional(cond)))
-
- case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
- u.copy(condition = Some(simplifyConditional(cond)))
-
- case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
- m.copy(
- mergeCondition = simplifyConditional(mergeCond),
- matchedActions = simplifyConditional(matchedActions),
- notMatchedActions = simplifyConditional(notMatchedActions))
- }
-
- private def simplifyConditional(e: Expression): Expression = e match {
- case And(left, right) => And(simplifyConditional(left), simplifyConditional(right))
- case Or(left, right) => Or(simplifyConditional(left), simplifyConditional(right))
- case If(cond, trueValue, FalseLiteral) => And(cond, trueValue)
- case If(cond, trueValue, TrueLiteral) => Or(Not(Coalesce(Seq(cond, FalseLiteral))), trueValue)
- case If(cond, FalseLiteral, falseValue) =>
- And(Not(Coalesce(Seq(cond, FalseLiteral))), falseValue)
- case If(cond, TrueLiteral, falseValue) => Or(cond, falseValue)
- case CaseWhen(Seq((cond, trueValue)),
- Some(FalseLiteral) | Some(Literal(null, BooleanType)) | None) =>
- And(cond, trueValue)
- case CaseWhen(Seq((cond, trueValue)), Some(TrueLiteral)) =>
- Or(Not(Coalesce(Seq(cond, FalseLiteral))), trueValue)
- case CaseWhen(Seq((cond, FalseLiteral)), Some(elseValue)) =>
- And(Not(Coalesce(Seq(cond, FalseLiteral))), elseValue)
- case CaseWhen(Seq((cond, TrueLiteral)), Some(elseValue)) =>
- Or(cond, elseValue)
- case e if e.dataType == BooleanType => e
- case e =>
- assert(e.dataType != BooleanType,
- "Expected a Boolean type expression in ExtendedSimplifyConditionalsInPredicate, " +
- s"but got the type `${e.dataType.catalogString}` in `${e.sql}`.")
- e
- }
-
- private def simplifyConditional(mergeActions: Seq[MergeAction]): Seq[MergeAction] = {
- mergeActions.map {
- case u @ UpdateAction(Some(cond), _) => u.copy(condition = Some(simplifyConditional(cond)))
- case u @ UpdateStarAction(Some(cond)) => u.copy(condition = Some(simplifyConditional(cond)))
- case d @ DeleteAction(Some(cond)) => d.copy(condition = Some(simplifyConditional(cond)))
- case i @ InsertAction(Some(cond), _) => i.copy(condition = Some(simplifyConditional(cond)))
- case i @ InsertStarAction(Some(cond)) => i.copy(condition = Some(simplifyConditional(cond)))
- case other => other
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
deleted file mode 100644
index f0324697e5..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.parser.extensions
-
-import java.util.Locale
-import org.antlr.v4.runtime._
-import org.antlr.v4.runtime.atn.PredictionMode
-import org.antlr.v4.runtime.misc.Interval
-import org.antlr.v4.runtime.misc.ParseCancellationException
-import org.antlr.v4.runtime.tree.TerminalNodeImpl
-import org.apache.iceberg.common.DynConstructors
-import org.apache.iceberg.spark.ExtendedParser
-import org.apache.iceberg.spark.ExtendedParser.RawOrderField
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.procedures.SparkProcedures
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext
-import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoContext
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoTable
-import org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.UpdateTable
-import org.apache.spark.sql.catalyst.trees.Origin
-import org.apache.spark.sql.connector.catalog.Table
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.execution.command.ExplainCommand
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.VariableSubstitution
-import org.apache.spark.sql.types.DataType
-import org.apache.spark.sql.types.StructType
-import scala.jdk.CollectionConverters._
-import scala.util.Try
-
-class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserInterface with ExtendedParser {
-
- import IcebergSparkSqlExtensionsParser._
-
- private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get)
- private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate)
-
- /**
- * Parse a string to a DataType.
- */
- override def parseDataType(sqlText: String): DataType = {
- delegate.parseDataType(sqlText)
- }
-
- /**
- * Parse a string to a raw DataType without CHAR/VARCHAR replacement.
- */
- def parseRawDataType(sqlText: String): DataType = throw new UnsupportedOperationException()
-
- /**
- * Parse a string to an Expression.
- */
- override def parseExpression(sqlText: String): Expression = {
- delegate.parseExpression(sqlText)
- }
-
- /**
- * Parse a string to a TableIdentifier.
- */
- override def parseTableIdentifier(sqlText: String): TableIdentifier = {
- delegate.parseTableIdentifier(sqlText)
- }
-
- /**
- * Parse a string to a FunctionIdentifier.
- */
- override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = {
- delegate.parseFunctionIdentifier(sqlText)
- }
-
- /**
- * Parse a string to a multi-part identifier.
- */
- override def parseMultipartIdentifier(sqlText: String): Seq[String] = {
- delegate.parseMultipartIdentifier(sqlText)
- }
-
- /**
- * Creates StructType for a given SQL string, which is a comma separated list of field
- * definitions which will preserve the correct Hive metadata.
- */
- override def parseTableSchema(sqlText: String): StructType = {
- delegate.parseTableSchema(sqlText)
- }
-
- override def parseSortOrder(sqlText: String): java.util.List[RawOrderField] = {
- val fields = parse(sqlText) { parser => astBuilder.visitSingleOrder(parser.singleOrder()) }
- fields.map { field =>
- val (term, direction, order) = field
- new RawOrderField(term, direction, order)
- }.asJava
- }
-
- /**
- * Parse a string to a LogicalPlan.
- */
- override def parsePlan(sqlText: String): LogicalPlan = {
- val sqlTextAfterSubstitution = substitutor.substitute(sqlText)
- if (isIcebergCommand(sqlTextAfterSubstitution)) {
- parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }.asInstanceOf[LogicalPlan]
- } else {
- val parsedPlan = delegate.parsePlan(sqlText)
- parsedPlan match {
- case e: ExplainCommand =>
- e.copy(logicalPlan = replaceRowLevelCommands(e.logicalPlan))
- case p =>
- replaceRowLevelCommands(p)
- }
- }
- }
-
- private def replaceRowLevelCommands(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
- case DeleteFromTable(UnresolvedIcebergTable(aliasedTable), condition) =>
- DeleteFromIcebergTable(aliasedTable, Some(condition))
-
- case UpdateTable(UnresolvedIcebergTable(aliasedTable), assignments, condition) =>
- UpdateIcebergTable(aliasedTable, assignments, condition)
-
- case MergeIntoTable(UnresolvedIcebergTable(aliasedTable), source, cond, matchedActions, notMatchedActions) =>
- // cannot construct MergeIntoIcebergTable right away as MERGE operations require special resolution
- // that's why the condition and actions must be hidden from the regular resolution rules in Spark
- // see ResolveMergeIntoTableReferences for details
- val context = MergeIntoContext(cond, matchedActions, notMatchedActions)
- UnresolvedMergeIntoIcebergTable(aliasedTable, source, context)
- }
-
- object UnresolvedIcebergTable {
-
- def unapply(plan: LogicalPlan): Option[LogicalPlan] = {
- EliminateSubqueryAliases(plan) match {
- case UnresolvedRelation(multipartIdentifier, _, _) if isIcebergTable(multipartIdentifier) =>
- Some(plan)
- case _ =>
- None
- }
- }
-
- private def isIcebergTable(multipartIdent: Seq[String]): Boolean = {
- val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(SparkSession.active, multipartIdent.asJava)
- catalogAndIdentifier.catalog match {
- case tableCatalog: TableCatalog =>
- Try(tableCatalog.loadTable(catalogAndIdentifier.identifier))
- .map(isIcebergTable)
- .getOrElse(false)
-
- case _ =>
- false
- }
- }
-
- private def isIcebergTable(table: Table): Boolean = table match {
- case _: SparkTable => true
- case _ => false
- }
- }
-
- private def isIcebergCommand(sqlText: String): Boolean = {
- val normalized = sqlText.toLowerCase(Locale.ROOT).trim()
- // Strip simple SQL comments that terminate a line, e.g. comments starting with `--` .
- .replaceAll("--.*?\\n", " ")
- // Strip newlines.
- .replaceAll("\\s+", " ")
- // Strip comments of the form /* ... */. This must come after stripping newlines so that
- // comments that span multiple lines are caught.
- .replaceAll("/\\*.*?\\*/", " ")
- // Strip backtick then `system`.`ancestors_of` changes to system.ancestors_of
- .replaceAll("`", "")
- .trim()
- isIcebergProcedure(normalized) || (
- normalized.startsWith("alter table") && (
- normalized.contains("add partition field") ||
- normalized.contains("drop partition field") ||
- normalized.contains("replace partition field") ||
- normalized.contains("write ordered by") ||
- normalized.contains("write locally ordered by") ||
- normalized.contains("write distributed by") ||
- normalized.contains("write unordered") ||
- normalized.contains("set identifier fields") ||
- normalized.contains("drop identifier fields") ||
- isSnapshotRefDdl(normalized)))
- }
-
- // All builtin Iceberg procedures are under the 'system' namespace
- private def isIcebergProcedure(normalized: String): Boolean = {
- normalized.startsWith("call") &&
- SparkProcedures.names().asScala.map("system." + _).exists(normalized.contains)
- }
-
- private def isSnapshotRefDdl(normalized: String): Boolean = {
- normalized.contains("create branch") ||
- normalized.contains("replace branch") ||
- normalized.contains("create tag") ||
- normalized.contains("replace tag") ||
- normalized.contains("drop branch") ||
- normalized.contains("drop tag")
- }
-
- protected def parse[T](command: String)(toResult: IcebergSqlExtensionsParser => T): T = {
- val lexer = new IcebergSqlExtensionsLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
- lexer.removeErrorListeners()
- lexer.addErrorListener(IcebergParseErrorListener)
-
- val tokenStream = new CommonTokenStream(lexer)
- val parser = new IcebergSqlExtensionsParser(tokenStream)
- parser.addParseListener(IcebergSqlExtensionsPostProcessor)
- parser.removeErrorListeners()
- parser.addErrorListener(IcebergParseErrorListener)
-
- // https://github.com/antlr/antlr4/issues/192#issuecomment-15238595
- // Save a great deal of time on correct inputs by using a two-stage parsing strategy.
- try {
- try {
- // first, try parsing with potentially faster SLL mode and BailErrorStrategy
- parser.setErrorHandler(new BailErrorStrategy)
- parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
- toResult(parser)
- }
- catch {
- case _: ParseCancellationException =>
- // if we fail, parse with LL mode with DefaultErrorStrategy
- tokenStream.seek(0) // rewind input stream
- parser.reset()
-
- // Try Again.
- parser.setErrorHandler(new DefaultErrorStrategy)
- parser.getInterpreter.setPredictionMode(PredictionMode.LL)
- toResult(parser)
- }
- }
- catch {
- case e: IcebergParseException if e.command.isDefined =>
- throw e
- case e: IcebergParseException =>
- throw e.withCommand(command)
- case e: AnalysisException =>
- val position = Origin(e.line, e.startPosition)
- throw new IcebergParseException(Option(command), e.message, position, position)
- }
- }
-
- override def parseQuery(sqlText: String): LogicalPlan = {
- parsePlan(sqlText)
- }
-}
-
-object IcebergSparkSqlExtensionsParser {
- private val substitutorCtor: DynConstructors.Ctor[VariableSubstitution] =
- DynConstructors.builder()
- .impl(classOf[VariableSubstitution])
- .impl(classOf[VariableSubstitution], classOf[SQLConf])
- .build()
-}
-
-/* Copied from Apache Spark's to avoid dependency on Spark Internals */
-class UpperCaseCharStream(wrapped: CodePointCharStream) extends CharStream {
- override def consume(): Unit = wrapped.consume
- override def getSourceName(): String = wrapped.getSourceName
- override def index(): Int = wrapped.index
- override def mark(): Int = wrapped.mark
- override def release(marker: Int): Unit = wrapped.release(marker)
- override def seek(where: Int): Unit = wrapped.seek(where)
- override def size(): Int = wrapped.size
-
- override def getText(interval: Interval): String = wrapped.getText(interval)
-
- // scalastyle:off
- override def LA(i: Int): Int = {
- val la = wrapped.LA(i)
- if (la == 0 || la == IntStream.EOF) la
- else Character.toUpperCase(la)
- }
- // scalastyle:on
-}
-
-/**
- * The post-processor validates & cleans-up the parse tree during the parse process.
- */
-case object IcebergSqlExtensionsPostProcessor extends IcebergSqlExtensionsBaseListener {
-
- /** Remove the back ticks from an Identifier. */
- override def exitQuotedIdentifier(ctx: QuotedIdentifierContext): Unit = {
- replaceTokenByIdentifier(ctx, 1) { token =>
- // Remove the double back ticks in the string.
- token.setText(token.getText.replace("``", "`"))
- token
- }
- }
-
- /** Treat non-reserved keywords as Identifiers. */
- override def exitNonReserved(ctx: NonReservedContext): Unit = {
- replaceTokenByIdentifier(ctx, 0)(identity)
- }
-
- private def replaceTokenByIdentifier(
- ctx: ParserRuleContext,
- stripMargins: Int)(
- f: CommonToken => CommonToken = identity): Unit = {
- val parent = ctx.getParent
- parent.removeLastChild()
- val token = ctx.getChild(0).getPayload.asInstanceOf[Token]
- val newToken = new CommonToken(
- new org.antlr.v4.runtime.misc.Pair(token.getTokenSource, token.getInputStream),
- IcebergSqlExtensionsParser.IDENTIFIER,
- token.getChannel,
- token.getStartIndex + stripMargins,
- token.getStopIndex - stripMargins)
- parent.addChild(new TerminalNodeImpl(f(newToken)))
- }
-}
-
-/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
-case object IcebergParseErrorListener extends BaseErrorListener {
- override def syntaxError(
- recognizer: Recognizer[_, _],
- offendingSymbol: scala.Any,
- line: Int,
- charPositionInLine: Int,
- msg: String,
- e: RecognitionException): Unit = {
- val (start, stop) = offendingSymbol match {
- case token: CommonToken =>
- val start = Origin(Some(line), Some(token.getCharPositionInLine))
- val length = token.getStopIndex - token.getStartIndex + 1
- val stop = Origin(Some(line), Some(token.getCharPositionInLine + length))
- (start, stop)
- case _ =>
- val start = Origin(Some(line), Some(charPositionInLine))
- (start, start)
- }
- throw new IcebergParseException(None, msg, start, stop)
- }
-}
-
-/**
- * Copied from Apache Spark
- * A [[ParseException]] is an [[AnalysisException]] that is thrown during the parse process. It
- * contains fields and an extended error message that make reporting and diagnosing errors easier.
- */
-class IcebergParseException(
- val command: Option[String],
- message: String,
- val start: Origin,
- val stop: Origin) extends AnalysisException(message, start.line, start.startPosition) {
-
- def this(message: String, ctx: ParserRuleContext) = {
- this(Option(IcebergParserUtils.command(ctx)),
- message,
- IcebergParserUtils.position(ctx.getStart),
- IcebergParserUtils.position(ctx.getStop))
- }
-
- override def getMessage: String = {
- val builder = new StringBuilder
- builder ++= "\n" ++= message
- start match {
- case Origin(
- Some(l), Some(p), Some(startIndex), Some(stopIndex), Some(sqlText), Some(objectType), Some(objectName)) =>
- builder ++= s"(line $l, pos $p)\n"
- command.foreach { cmd =>
- val (above, below) = cmd.split("\n").splitAt(l)
- builder ++= "\n== SQL ==\n"
- above.foreach(builder ++= _ += '\n')
- builder ++= (0 until p).map(_ => "-").mkString("") ++= "^^^\n"
- below.foreach(builder ++= _ += '\n')
- }
- case _ =>
- command.foreach { cmd =>
- builder ++= "\n== SQL ==\n" ++= cmd
- }
- }
- builder.toString
- }
-
- def withCommand(cmd: String): IcebergParseException = {
- new IcebergParseException(Option(cmd), message, start, stop)
- }
-}
\ No newline at end of file
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
deleted file mode 100644
index 2e438de2b8..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.parser.extensions
-
-import java.util.Locale
-import java.util.concurrent.TimeUnit
-import org.antlr.v4.runtime._
-import org.antlr.v4.runtime.misc.Interval
-import org.antlr.v4.runtime.tree.ParseTree
-import org.antlr.v4.runtime.tree.TerminalNode
-import org.apache.iceberg.DistributionMode
-import org.apache.iceberg.NullOrder
-import org.apache.iceberg.SortDirection
-import org.apache.iceberg.expressions.Term
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergParserUtils.withOrigin
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser._
-import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.BranchOptions
-import org.apache.spark.sql.catalyst.plans.logical.CallArgument
-import org.apache.spark.sql.catalyst.plans.logical.CallStatement
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag
-import org.apache.spark.sql.catalyst.plans.logical.DropBranch
-import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.DropTag
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.NamedArgument
-import org.apache.spark.sql.catalyst.plans.logical.PositionalArgument
-import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
-import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
-import org.apache.spark.sql.catalyst.plans.logical.TagOptions
-import org.apache.spark.sql.catalyst.trees.CurrentOrigin
-import org.apache.spark.sql.catalyst.trees.Origin
-import org.apache.spark.sql.connector.expressions
-import org.apache.spark.sql.connector.expressions.ApplyTransform
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.IdentityTransform
-import org.apache.spark.sql.connector.expressions.LiteralValue
-import org.apache.spark.sql.connector.expressions.Transform
-import scala.jdk.CollectionConverters._
-
-class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
-
- private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] = list.asScala
- private def toSeq[T](list: java.util.List[T]): Seq[T] = toBuffer(list).toSeq
-
- /**
- * Create a [[CallStatement]] for a stored procedure call.
- */
- override def visitCall(ctx: CallContext): CallStatement = withOrigin(ctx) {
- val name = toSeq(ctx.multipartIdentifier.parts).map(_.getText)
- val args = toSeq(ctx.callArgument).map(typedVisit[CallArgument])
- CallStatement(name, args)
- }
-
- /**
- * Create an ADD PARTITION FIELD logical command.
- */
- override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField = withOrigin(ctx) {
- AddPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform),
- Option(ctx.name).map(_.getText))
- }
-
- /**
- * Create a DROP PARTITION FIELD logical command.
- */
- override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField = withOrigin(ctx) {
- DropPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform))
- }
-
- /**
- * Create a CREATE OR REPLACE BRANCH logical command.
- */
- override def visitCreateOrReplaceBranch(ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
- val createOrReplaceBranchClause = ctx.createReplaceBranchClause()
-
- val branchName = createOrReplaceBranchClause.identifier()
- val branchOptionsContext = Option(createOrReplaceBranchClause.branchOptions())
- val snapshotId = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotId()))
- .map(_.getText.toLong)
- val snapshotRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
- val minSnapshotsToKeep = snapshotRetention.flatMap(retention => Option(retention.minSnapshotsToKeep()))
- .map(minSnapshots => minSnapshots.number().getText.toLong)
- val maxSnapshotAgeMs = snapshotRetention
- .flatMap(retention => Option(retention.maxSnapshotAge()))
- .map(retention => TimeUnit.valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
- .toMillis(retention.number().getText.toLong))
- val branchRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
- val branchRefAgeMs = branchRetention.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
- val create = createOrReplaceBranchClause.CREATE() != null
- val replace = ctx.createReplaceBranchClause().REPLACE() != null
- val ifNotExists = createOrReplaceBranchClause.EXISTS() != null
-
- val branchOptions = BranchOptions(
- snapshotId,
- minSnapshotsToKeep,
- maxSnapshotAgeMs,
- branchRefAgeMs
- )
-
- CreateOrReplaceBranch(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- branchName.getText,
- branchOptions,
- create,
- replace,
- ifNotExists)
- }
-
- /**
- * Create an CREATE OR REPLACE TAG logical command.
- */
- override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag = withOrigin(ctx) {
- val createTagClause = ctx.createReplaceTagClause()
-
- val tagName = createTagClause.identifier().getText
-
- val tagOptionsContext = Option(createTagClause.tagOptions())
- val snapshotId = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.snapshotId()))
- .map(_.getText.toLong)
- val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
- val tagRefAgeMs = tagRetain.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
- val tagOptions = TagOptions(
- snapshotId,
- tagRefAgeMs
- )
-
- val create = createTagClause.CREATE() != null
- val replace = createTagClause.REPLACE() != null
- val ifNotExists = createTagClause.EXISTS() != null
-
- CreateOrReplaceTag(typedVisit[Seq[String]](ctx.multipartIdentifier),
- tagName,
- tagOptions,
- create,
- replace,
- ifNotExists)
- }
-
- /**
- * Create an DROP BRANCH logical command.
- */
- override def visitDropBranch(ctx: DropBranchContext): DropBranch = withOrigin(ctx) {
- DropBranch(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
- }
-
- /**
- * Create an DROP TAG logical command.
- */
- override def visitDropTag(ctx: DropTagContext): DropTag = withOrigin(ctx) {
- DropTag(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
- }
-
- /**
- * Create an REPLACE PARTITION FIELD logical command.
- */
- override def visitReplacePartitionField(ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
- ReplacePartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform(0)),
- typedVisit[Transform](ctx.transform(1)),
- Option(ctx.name).map(_.getText))
- }
-
- /**
- * Create an SET IDENTIFIER FIELDS logical command.
- */
- override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields = withOrigin(ctx) {
- SetIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
-
- /**
- * Create an DROP IDENTIFIER FIELDS logical command.
- */
- override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields = withOrigin(ctx) {
- DropIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
-
- /**
- * Create a [[SetWriteDistributionAndOrdering]] for changing the write distribution and ordering.
- */
- override def visitSetWriteDistributionAndOrdering(
- ctx: SetWriteDistributionAndOrderingContext): SetWriteDistributionAndOrdering = {
-
- val tableName = typedVisit[Seq[String]](ctx.multipartIdentifier)
-
- val (distributionSpec, orderingSpec) = toDistributionAndOrderingSpec(ctx.writeSpec)
-
- if (distributionSpec == null && orderingSpec == null) {
- throw new AnalysisException(
- "ALTER TABLE has no changes: missing both distribution and ordering clauses")
- }
-
- val distributionMode = if (distributionSpec != null) {
- DistributionMode.HASH
- } else if (orderingSpec.UNORDERED != null || orderingSpec.LOCALLY != null) {
- DistributionMode.NONE
- } else {
- DistributionMode.RANGE
- }
-
- val ordering = if (orderingSpec != null && orderingSpec.order != null) {
- toSeq(orderingSpec.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
- } else {
- Seq.empty
- }
-
- SetWriteDistributionAndOrdering(tableName, distributionMode, ordering)
- }
-
- private def toDistributionAndOrderingSpec(
- writeSpec: WriteSpecContext): (WriteDistributionSpecContext, WriteOrderingSpecContext) = {
-
- if (writeSpec.writeDistributionSpec.size > 1) {
- throw new AnalysisException("ALTER TABLE contains multiple distribution clauses")
- }
-
- if (writeSpec.writeOrderingSpec.size > 1) {
- throw new AnalysisException("ALTER TABLE contains multiple ordering clauses")
- }
-
- val distributionSpec = toBuffer(writeSpec.writeDistributionSpec).headOption.orNull
- val orderingSpec = toBuffer(writeSpec.writeOrderingSpec).headOption.orNull
-
- (distributionSpec, orderingSpec)
- }
-
- /**
- * Create an order field.
- */
- override def visitOrderField(ctx: OrderFieldContext): (Term, SortDirection, NullOrder) = {
- val term = Spark3Util.toIcebergTerm(typedVisit[Transform](ctx.transform))
- val direction = Option(ctx.ASC).map(_ => SortDirection.ASC)
- .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
- .getOrElse(SortDirection.ASC)
- val nullOrder = Option(ctx.FIRST).map(_ => NullOrder.NULLS_FIRST)
- .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
- .getOrElse(if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
- (term, direction, nullOrder)
- }
-
- /**
- * Create an IdentityTransform for a column reference.
- */
- override def visitIdentityTransform(ctx: IdentityTransformContext): Transform = withOrigin(ctx) {
- IdentityTransform(FieldReference(typedVisit[Seq[String]](ctx.multipartIdentifier())))
- }
-
- /**
- * Create a named Transform from argument expressions.
- */
- override def visitApplyTransform(ctx: ApplyTransformContext): Transform = withOrigin(ctx) {
- val args = toSeq(ctx.arguments).map(typedVisit[expressions.Expression])
- ApplyTransform(ctx.transformName.getText, args)
- }
-
- /**
- * Create a transform argument from a column reference or a constant.
- */
- override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression = withOrigin(ctx) {
- val reference = Option(ctx.multipartIdentifier())
- .map(typedVisit[Seq[String]])
- .map(FieldReference(_))
- val literal = Option(ctx.constant)
- .map(visitConstant)
- .map(lit => LiteralValue(lit.value, lit.dataType))
- reference.orElse(literal)
- .getOrElse(throw new IcebergParseException(s"Invalid transform argument", ctx))
- }
-
- /**
- * Return a multi-part identifier as Seq[String].
- */
- override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) {
- toSeq(ctx.parts).map(_.getText)
- }
-
- override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] = withOrigin(ctx) {
- toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
- }
-
- /**
- * Create a positional argument in a stored procedure call.
- */
- override def visitPositionalArgument(ctx: PositionalArgumentContext): CallArgument = withOrigin(ctx) {
- val expr = typedVisit[Expression](ctx.expression)
- PositionalArgument(expr)
- }
-
- /**
- * Create a named argument in a stored procedure call.
- */
- override def visitNamedArgument(ctx: NamedArgumentContext): CallArgument = withOrigin(ctx) {
- val name = ctx.identifier.getText
- val expr = typedVisit[Expression](ctx.expression)
- NamedArgument(name, expr)
- }
-
- override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) {
- visit(ctx.statement).asInstanceOf[LogicalPlan]
- }
-
- def visitConstant(ctx: ConstantContext): Literal = {
- delegate.parseExpression(ctx.getText).asInstanceOf[Literal]
- }
-
- override def visitExpression(ctx: ExpressionContext): Expression = {
- // reconstruct the SQL string and parse it using the main Spark parser
- // while we can avoid the logic to build Spark expressions, we still have to parse them
- // we cannot call ctx.getText directly since it will not render spaces correctly
- // that's why we need to recurse down the tree in reconstructSqlString
- val sqlString = reconstructSqlString(ctx)
- delegate.parseExpression(sqlString)
- }
-
- private def reconstructSqlString(ctx: ParserRuleContext): String = {
- toBuffer(ctx.children).map {
- case c: ParserRuleContext => reconstructSqlString(c)
- case t: TerminalNode => t.getText
- }.mkString(" ")
- }
-
- private def typedVisit[T](ctx: ParseTree): T = {
- ctx.accept(this).asInstanceOf[T]
- }
-}
-
-/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
-object IcebergParserUtils {
-
- private[sql] def withOrigin[T](ctx: ParserRuleContext)(f: => T): T = {
- val current = CurrentOrigin.get
- CurrentOrigin.set(position(ctx.getStart))
- try {
- f
- } finally {
- CurrentOrigin.set(current)
- }
- }
-
- private[sql] def position(token: Token): Origin = {
- val opt = Option(token)
- Origin(opt.map(_.getLine), opt.map(_.getCharPositionInLine))
- }
-
- /** Get the command which created the token. */
- private[sql] def command(ctx: ParserRuleContext): String = {
- val stream = ctx.getStart.getInputStream
- stream.getText(Interval.of(0, stream.size() - 1))
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
deleted file mode 100644
index f45e44e2ce..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.planning
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.RowLevelCommand
-import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
-import org.apache.spark.sql.connector.catalog.Table
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
-
-/**
- * An extractor for operations such as DELETE and MERGE that require rewriting data.
- *
- * This class extracts the following entities:
- * - the row-level command (such as DeleteFromIcebergTable);
- * - the read relation in the rewrite plan that can be either DataSourceV2Relation or
- * DataSourceV2ScanRelation depending on whether the planning has already happened;
- * - the current rewrite plan.
- */
-object RewrittenRowLevelCommand {
- type ReturnType = (RowLevelCommand, LogicalPlan, LogicalPlan)
-
- def unapply(plan: LogicalPlan): Option[ReturnType] = plan match {
- case c: RowLevelCommand if c.rewritePlan.nonEmpty =>
- val rewritePlan = c.rewritePlan.get
-
- // both ReplaceData and WriteDelta reference a write relation
- // but the corresponding read relation should be at the bottom of the write plan
- // both the write and read relations will share the same RowLevelOperationTable object
- // that's why it is safe to use reference equality to find the needed read relation
-
- val allowScanDuplication = c match {
- // group-based updates that rely on the union approach may have multiple identical scans
- case _: UpdateIcebergTable if rewritePlan.isInstanceOf[ReplaceIcebergData] => true
- case _ => false
- }
-
- rewritePlan match {
- case rd @ ReplaceIcebergData(DataSourceV2Relation(table, _, _, _, _), query, _, _) =>
- val readRelation = findReadRelation(table, query, allowScanDuplication)
- readRelation.map((c, _, rd))
- case wd @ WriteDelta(DataSourceV2Relation(table, _, _, _, _), query, _, _, _) =>
- val readRelation = findReadRelation(table, query, allowScanDuplication)
- readRelation.map((c, _, wd))
- case _ =>
- None
- }
-
- case _ =>
- None
- }
-
- private def findReadRelation(
- table: Table,
- plan: LogicalPlan,
- allowScanDuplication: Boolean): Option[LogicalPlan] = {
-
- val readRelations = plan.collect {
- case r: DataSourceV2Relation if r.table eq table => r
- case r: DataSourceV2ScanRelation if r.relation.table eq table => r
- }
-
- // in some cases, the optimizer replaces the v2 read relation with a local relation
- // for example, there is no reason to query the table if the condition is always false
- // that's why it is valid not to find the corresponding v2 read relation
-
- readRelations match {
- case relations if relations.isEmpty =>
- None
-
- case Seq(relation) =>
- Some(relation)
-
- case Seq(relation1: DataSourceV2Relation, relation2: DataSourceV2Relation)
- if allowScanDuplication && (relation1.table eq relation2.table) =>
- Some(relation1)
-
- case Seq(relation1: DataSourceV2ScanRelation, relation2: DataSourceV2ScanRelation)
- if allowScanDuplication && (relation1.scan eq relation2.scan) =>
- Some(relation1)
-
- case Seq(relation1, relation2) if allowScanDuplication =>
- throw new AnalysisException(s"Row-level read relations don't match: $relation1, $relation2")
-
- case relations if allowScanDuplication =>
- throw new AnalysisException(s"Expected up to two row-level read relations: $relations")
-
- case relations =>
- throw new AnalysisException(s"Expected only one row-level read relation: $relations")
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
deleted file mode 100644
index e8b1b29411..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"AddPartitionField ${table.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
deleted file mode 100644
index 4d7e0a086b..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-case class BranchOptions (snapshotId: Option[Long], numSnapshots: Option[Long],
- snapshotRetain: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
deleted file mode 100644
index 9616dae5a8..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.connector.iceberg.catalog.Procedure
-
-case class Call(procedure: Procedure, args: Seq[Expression]) extends LeafCommand {
- override lazy val output: Seq[Attribute] = procedure.outputType.toAttributes
-
- override def simpleString(maxFields: Int): String = {
- s"Call${truncatedString(output.toSeq, "[", ", ", "]", maxFields)} ${procedure.description}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
deleted file mode 100644
index b7981a3c7a..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class CreateOrReplaceBranch(
- table: Seq[String],
- branch: String,
- branchOptions: BranchOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"CreateOrReplaceBranch branch: ${branch} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
deleted file mode 100644
index 6e7db84a90..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class CreateOrReplaceTag(
- table: Seq[String],
- tag: String,
- tagOptions: TagOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"CreateOrReplaceTag tag: ${tag} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromIcebergTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromIcebergTable.scala
deleted file mode 100644
index d1268e416c..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromIcebergTable.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Expression
-
-case class DeleteFromIcebergTable(
- table: LogicalPlan,
- condition: Option[Expression],
- rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
-
- override def children: Seq[LogicalPlan] = if (rewritePlan.isDefined) {
- table :: rewritePlan.get :: Nil
- } else {
- table :: Nil
- }
-
- override def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand = {
- copy(rewritePlan = Some(newRewritePlan))
- }
-
- override protected def withNewChildrenInternal(
- newChildren: IndexedSeq[LogicalPlan]): DeleteFromIcebergTable = {
- if (newChildren.size == 1) {
- copy(table = newChildren.head, rewritePlan = None)
- } else {
- require(newChildren.size == 2, "DeleteFromIcebergTable expects either one or two children")
- val Seq(newTable, newRewritePlan) = newChildren.take(2)
- copy(table = newTable, rewritePlan = Some(newRewritePlan))
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
deleted file mode 100644
index bee0b0fae6..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class DropBranch(table: Seq[String], branch: String, ifExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropBranch branch: ${branch} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
deleted file mode 100644
index 29dd686a0f..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class DropIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropIdentifierFields ${table.quoted} (${fields.quoted})"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
deleted file mode 100644
index fb14513241..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class DropPartitionField(table: Seq[String], transform: Transform) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropPartitionField ${table.quoted} ${transform.describe}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
deleted file mode 100644
index 7e4b38e74d..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class DropTag(table: Seq[String], tag: String, ifExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropTag tag: ${tag} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
deleted file mode 100644
index 8f84851dcd..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
-import org.apache.spark.sql.catalyst.expressions.Expression
-
-case class MergeIntoIcebergTable(
- targetTable: LogicalPlan,
- sourceTable: LogicalPlan,
- mergeCondition: Expression,
- matchedActions: Seq[MergeAction],
- notMatchedActions: Seq[MergeAction],
- rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
-
- lazy val aligned: Boolean = {
- val matchedActionsAligned = matchedActions.forall {
- case UpdateAction(_, assignments) =>
- AssignmentUtils.aligned(targetTable, assignments)
- case _: DeleteAction =>
- true
- case _ =>
- false
- }
-
- val notMatchedActionsAligned = notMatchedActions.forall {
- case InsertAction(_, assignments) =>
- AssignmentUtils.aligned(targetTable, assignments)
- case _ =>
- false
- }
-
- matchedActionsAligned && notMatchedActionsAligned
- }
-
- def condition: Option[Expression] = Some(mergeCondition)
-
- override def children: Seq[LogicalPlan] = if (rewritePlan.isDefined) {
- targetTable :: sourceTable :: rewritePlan.get :: Nil
- } else {
- targetTable :: sourceTable :: Nil
- }
-
- override def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand = {
- copy(rewritePlan = Some(newRewritePlan))
- }
-
- override protected def withNewChildrenInternal(
- newChildren: IndexedSeq[LogicalPlan]): MergeIntoIcebergTable = {
-
- newChildren match {
- case Seq(newTarget, newSource) =>
- copy(targetTable = newTarget, sourceTable = newSource, rewritePlan = None)
- case Seq(newTarget, newSource, newRewritePlan) =>
- copy(targetTable = newTarget, sourceTable = newSource, rewritePlan = Some(newRewritePlan))
- case _ =>
- throw new IllegalArgumentException("MergeIntoIcebergTable expects either two or three children")
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
deleted file mode 100644
index 9c17ef02e6..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.util.truncatedString
-
-case class MergeRows(
- isSourceRowPresent: Expression,
- isTargetRowPresent: Expression,
- matchedConditions: Seq[Expression],
- matchedOutputs: Seq[Seq[Expression]],
- notMatchedConditions: Seq[Expression],
- notMatchedOutputs: Seq[Seq[Expression]],
- targetOutput: Seq[Expression],
- performCardinalityCheck: Boolean,
- emitNotMatchedTargetRows: Boolean,
- output: Seq[Attribute],
- child: LogicalPlan) extends UnaryNode {
-
- require(targetOutput.nonEmpty || !emitNotMatchedTargetRows)
-
- override lazy val producedAttributes: AttributeSet = {
- AttributeSet(output.filterNot(attr => inputSet.contains(attr)))
- }
-
- override lazy val references: AttributeSet = child.outputSet
-
- override def simpleString(maxFields: Int): String = {
- s"MergeRows${truncatedString(output, "[", ", ", "]", maxFields)}"
- }
-
- override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = {
- copy(child = newChild)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala
deleted file mode 100644
index c21df71f06..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class NoStatsUnaryNode(child: LogicalPlan) extends UnaryNode {
- override def output: Seq[Attribute] = child.output
- override def stats: Statistics = Statistics(Long.MaxValue)
-
- override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = {
- copy(child = newChild)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceIcebergData.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceIcebergData.scala
deleted file mode 100644
index 2b741bef12..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceIcebergData.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.analysis.NamedRelation
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-import org.apache.spark.sql.catalyst.util.CharVarcharUtils
-import org.apache.spark.sql.connector.write.Write
-import org.apache.spark.sql.types.DataType
-
-/**
- * Replace data in an existing table.
- */
-case class ReplaceIcebergData(
- table: NamedRelation,
- query: LogicalPlan,
- originalTable: NamedRelation,
- write: Option[Write] = None) extends V2WriteCommandLike {
-
- override lazy val references: AttributeSet = query.outputSet
- override lazy val stringArgs: Iterator[Any] = Iterator(table, query, write)
-
- // the incoming query may include metadata columns
- lazy val dataInput: Seq[Attribute] = {
- val tableAttrNames = table.output.map(_.name)
- query.output.filter(attr => tableAttrNames.exists(conf.resolver(_, attr.name)))
- }
-
- override def outputResolved: Boolean = {
- assert(table.resolved && query.resolved,
- "`outputResolved` can only be called when `table` and `query` are both resolved.")
-
- // take into account only incoming data columns and ignore metadata columns in the query
- // they will be discarded after the logical write is built in the optimizer
- // metadata columns may be needed to request a correct distribution or ordering
- // but are not passed back to the data source during writes
-
- table.skipSchemaResolution || (dataInput.size == table.output.size &&
- dataInput.zip(table.output).forall { case (inAttr, outAttr) =>
- val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
- // names and types must match, nullability must be compatible
- inAttr.name == outAttr.name &&
- DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outType) &&
- (outAttr.nullable || !inAttr.nullable)
- })
- }
-
- override protected def withNewChildInternal(newChild: LogicalPlan): ReplaceIcebergData = {
- copy(query = newChild)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
deleted file mode 100644
index 8c660c6f37..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class ReplacePartitionField(
- table: Seq[String],
- transformFrom: Transform,
- transformTo: Transform,
- name: Option[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"ReplacePartitionField ${table.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala
deleted file mode 100644
index 837ee963bc..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Expression
-
-trait RowLevelCommand extends Command with SupportsSubquery {
- def condition: Option[Expression]
- def rewritePlan: Option[LogicalPlan]
- def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
deleted file mode 100644
index a5fa28a617..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class SetIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"SetIdentifierFields ${table.quoted} (${fields.quoted})"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
deleted file mode 100644
index 85e3b95f4a..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-case class TagOptions(snapshotId: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala
deleted file mode 100644
index 895aa733ff..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Expression
-
-/**
- * A node that hides the MERGE condition and actions from regular Spark resolution.
- */
-case class UnresolvedMergeIntoIcebergTable(
- targetTable: LogicalPlan,
- sourceTable: LogicalPlan,
- context: MergeIntoContext) extends BinaryCommand {
-
- def duplicateResolved: Boolean = targetTable.outputSet.intersect(sourceTable.outputSet).isEmpty
-
- override def left: LogicalPlan = targetTable
- override def right: LogicalPlan = sourceTable
-
- override protected def withNewChildrenInternal(newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan = {
- copy(targetTable = newLeft, sourceTable = newRight)
- }
-}
-
-case class MergeIntoContext(
- mergeCondition: Expression,
- matchedActions: Seq[MergeAction],
- notMatchedActions: Seq[MergeAction])
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala
deleted file mode 100644
index 790eb9380e..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
-import org.apache.spark.sql.catalyst.expressions.Expression
-
-case class UpdateIcebergTable(
- table: LogicalPlan,
- assignments: Seq[Assignment],
- condition: Option[Expression],
- rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
-
- lazy val aligned: Boolean = AssignmentUtils.aligned(table, assignments)
-
- override def children: Seq[LogicalPlan] = if (rewritePlan.isDefined) {
- table :: rewritePlan.get :: Nil
- } else {
- table :: Nil
- }
-
- override def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand = {
- copy(rewritePlan = Some(newRewritePlan))
- }
-
- override protected def withNewChildrenInternal(
- newChildren: IndexedSeq[LogicalPlan]): UpdateIcebergTable = {
- if (newChildren.size == 1) {
- copy(table = newChildren.head, rewritePlan = None)
- } else {
- require(newChildren.size == 2, "UpdateTable expects either one or two children")
- val Seq(newTable, newRewritePlan) = newChildren.take(2)
- copy(table = newTable, rewritePlan = Some(newRewritePlan))
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
deleted file mode 100644
index 9192d74b7c..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.analysis.NamedRelation
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-
-// a node similar to V2WriteCommand in Spark but does not extend Command
-// as ReplaceData and WriteDelta that extend this trait are nested within other commands
-trait V2WriteCommandLike extends UnaryNode {
- def table: NamedRelation
- def query: LogicalPlan
- def outputResolved: Boolean
-
- override lazy val resolved: Boolean = table.resolved && query.resolved && outputResolved
-
- override def child: LogicalPlan = query
- override def output: Seq[Attribute] = Seq.empty
- override def producedAttributes: AttributeSet = outputSet
- // Commands are eagerly executed. They will be converted to LocalRelation after the DataFrame
- // is created. That said, the statistics of a command is useless. Here we just return a dummy
- // statistics to avoid unnecessary statistics calculation of command's children.
- override def stats: Statistics = Statistics.DUMMY
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteDelta.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteDelta.scala
deleted file mode 100644
index 534427cc04..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteDelta.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
-import org.apache.spark.sql.catalyst.analysis.NamedRelation
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
-import org.apache.spark.sql.catalyst.expressions.NamedExpression
-import org.apache.spark.sql.catalyst.util.CharVarcharUtils
-import org.apache.spark.sql.catalyst.util.RowDeltaUtils.OPERATION_COLUMN
-import org.apache.spark.sql.catalyst.util.WriteDeltaProjections
-import org.apache.spark.sql.connector.iceberg.write.DeltaWrite
-import org.apache.spark.sql.connector.iceberg.write.SupportsDelta
-import org.apache.spark.sql.connector.write.RowLevelOperationTable
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.types.DataType
-import org.apache.spark.sql.types.IntegerType
-import org.apache.spark.sql.types.StructField
-
-/**
- * Writes a delta of rows to an existing table.
- */
-case class WriteDelta(
- table: NamedRelation,
- query: LogicalPlan,
- originalTable: NamedRelation,
- projections: WriteDeltaProjections,
- write: Option[DeltaWrite] = None) extends V2WriteCommandLike {
-
- override protected lazy val stringArgs: Iterator[Any] = Iterator(table, query, write)
-
- private def operationResolved: Boolean = {
- val attr = query.output.head
- attr.name == OPERATION_COLUMN && attr.dataType == IntegerType && !attr.nullable
- }
-
- private def operation: SupportsDelta = {
- EliminateSubqueryAliases(table) match {
- case DataSourceV2Relation(RowLevelOperationTable(_, operation), _, _, _, _) =>
- operation match {
- case supportsDelta: SupportsDelta =>
- supportsDelta
- case _ =>
- throw new AnalysisException(s"Operation $operation is not a delta operation")
- }
- case _ =>
- throw new AnalysisException(s"Cannot retrieve row-level operation from $table")
- }
- }
-
- private def rowAttrsResolved: Boolean = {
- table.skipSchemaResolution || (projections.rowProjection match {
- case Some(projection) =>
- table.output.size == projection.schema.size &&
- projection.schema.zip(table.output).forall { case (field, outAttr) =>
- isCompatible(field, outAttr)
- }
- case None =>
- true
- })
- }
-
- private def rowIdAttrsResolved: Boolean = {
- val rowIdAttrs = ExtendedV2ExpressionUtils.resolveRefs[AttributeReference](
- operation.rowId.toSeq,
- originalTable)
-
- projections.rowIdProjection.schema.forall { field =>
- rowIdAttrs.exists(rowIdAttr => isCompatible(field, rowIdAttr))
- }
- }
-
- private def metadataAttrsResolved: Boolean = {
- projections.metadataProjection match {
- case Some(projection) =>
- val metadataAttrs = ExtendedV2ExpressionUtils.resolveRefs[AttributeReference](
- operation.requiredMetadataAttributes.toSeq,
- originalTable)
-
- projection.schema.forall { field =>
- metadataAttrs.exists(metadataAttr => isCompatible(field, metadataAttr))
- }
- case None =>
- true
- }
- }
-
- private def isCompatible(projectionField: StructField, outAttr: NamedExpression): Boolean = {
- val inType = CharVarcharUtils.getRawType(projectionField.metadata).getOrElse(outAttr.dataType)
- val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
- // names and types must match, nullability must be compatible
- projectionField.name == outAttr.name &&
- DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
- (outAttr.nullable || !projectionField.nullable)
- }
-
- override def outputResolved: Boolean = {
- assert(table.resolved && query.resolved,
- "`outputResolved` can only be called when `table` and `query` are both resolved.")
-
- operationResolved && rowAttrsResolved && rowIdAttrsResolved && metadataAttrsResolved
- }
-
- override protected def withNewChildInternal(newChild: LogicalPlan): WriteDelta = {
- copy(query = newChild)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
deleted file mode 100644
index be15f32bc1..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Expression
-
-/**
- * A CALL statement, as parsed from SQL.
- */
-case class CallStatement(name: Seq[String], args: Seq[CallArgument]) extends LeafParsedStatement
-
-/**
- * An argument in a CALL statement.
- */
-sealed trait CallArgument {
- def expr: Expression
-}
-
-/**
- * An argument in a CALL statement identified by name.
- */
-case class NamedArgument(name: String, expr: Expression) extends CallArgument
-
-/**
- * An argument in a CALL statement identified by position.
- */
-case class PositionalArgument(expr: Expression) extends CallArgument
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala
deleted file mode 100644
index 9f8f07b77d..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.util
-
-object RowDeltaUtils {
- final val OPERATION_COLUMN: String = "__row_operation"
- final val DELETE_OPERATION: Int = 1
- final val UPDATE_OPERATION: Int = 2
- final val INSERT_OPERATION: Int = 3
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala
deleted file mode 100644
index e4e8b147d1..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.util
-
-import org.apache.spark.sql.catalyst.ProjectingInternalRow
-
-case class WriteDeltaProjections(
- rowProjection: Option[ProjectingInternalRow],
- rowIdProjection: ProjectingInternalRow,
- metadataProjection: Option[ProjectingInternalRow])
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/connector/expressions/TruncateTransform.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/connector/expressions/TruncateTransform.scala
deleted file mode 100644
index 2a3269e2db..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/connector/expressions/TruncateTransform.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.connector.expressions
-
-import org.apache.spark.sql.types.IntegerType
-
-private[sql] object TruncateTransform {
- def unapply(expr: Expression): Option[(Int, FieldReference)] = expr match {
- case transform: Transform =>
- transform match {
- case NamedTransform("truncate", Seq(Ref(seq: Seq[String]), Lit(value: Int, IntegerType))) =>
- Some((value, FieldReference(seq)))
- case NamedTransform("truncate", Seq(Lit(value: Int, IntegerType), Ref(seq: Seq[String]))) =>
- Some((value, FieldReference(seq)))
- case _ =>
- None
- }
- case _ =>
- None
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/ExtendedLogicalWriteInfoImpl.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/ExtendedLogicalWriteInfoImpl.scala
deleted file mode 100644
index 0ae10ada5a..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/ExtendedLogicalWriteInfoImpl.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.connector.write
-
-import org.apache.spark.sql.connector.iceberg.write.ExtendedLogicalWriteInfo
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.util.CaseInsensitiveStringMap
-
-private[sql] case class ExtendedLogicalWriteInfoImpl(
- queryId: String,
- schema: StructType,
- options: CaseInsensitiveStringMap,
- rowIdSchema: StructType = null,
- metadataSchema: StructType = null) extends ExtendedLogicalWriteInfo
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
deleted file mode 100644
index 55f327f7e4..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class AddPartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transform: Transform,
- name: Option[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- iceberg.table.updateSpec()
- .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
- .commit()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot add partition field to non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"AddPartitionField ${catalog.name}.${ident.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
deleted file mode 100644
index f66962a8c4..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.connector.iceberg.catalog.Procedure
-import scala.collection.compat.immutable.ArraySeq
-
-case class CallExec(
- output: Seq[Attribute],
- procedure: Procedure,
- input: InternalRow) extends LeafV2CommandExec {
-
- override protected def run(): Seq[InternalRow] = {
- ArraySeq.unsafeWrapArray(procedure.call(input))
- }
-
- override def simpleString(maxFields: Int): String = {
- s"CallExec${truncatedString(output, "[", ", ", "]", maxFields)} ${procedure.description}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
deleted file mode 100644
index 2be406e7f3..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.BranchOptions
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class CreateOrReplaceBranchExec(
- catalog: TableCatalog,
- ident: Identifier,
- branch: String,
- branchOptions: BranchOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val snapshotId: java.lang.Long = branchOptions.snapshotId
- .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId()))
- .map(java.lang.Long.valueOf)
- .orNull
-
- val manageSnapshots = iceberg.table().manageSnapshots()
- val refExists = null != iceberg.table().refs().get(branch)
-
- def safeCreateBranch(): Unit = {
- if (snapshotId == null) {
- manageSnapshots.createBranch(branch)
- } else {
- manageSnapshots.createBranch(branch, snapshotId)
- }
- }
-
- if (create && replace && !refExists) {
- safeCreateBranch()
- } else if (replace) {
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete replace branch operation on %s, main has no snapshot", ident)
- manageSnapshots.replaceBranch(branch, snapshotId)
- } else {
- if (refExists && ifNotExists) {
- return Nil
- }
-
- safeCreateBranch()
- }
-
- if (branchOptions.numSnapshots.nonEmpty) {
- manageSnapshots.setMinSnapshotsToKeep(branch, branchOptions.numSnapshots.get.toInt)
- }
-
- if (branchOptions.snapshotRetain.nonEmpty) {
- manageSnapshots.setMaxSnapshotAgeMs(branch, branchOptions.snapshotRetain.get)
- }
-
- if (branchOptions.snapshotRefRetain.nonEmpty) {
- manageSnapshots.setMaxRefAgeMs(branch, branchOptions.snapshotRefRetain.get)
- }
-
- manageSnapshots.commit()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot create or replace branch on non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"CreateOrReplace branch: $branch for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
deleted file mode 100644
index 372cd75486..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.TagOptions
-import org.apache.spark.sql.connector.catalog._
-
-case class CreateOrReplaceTagExec(
- catalog: TableCatalog,
- ident: Identifier,
- tag: String,
- tagOptions: TagOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val snapshotId: java.lang.Long = tagOptions.snapshotId
- .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId()))
- .map(java.lang.Long.valueOf)
- .orNull
-
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete create or replace tag operation on %s, main has no snapshot", ident)
-
- val manageSnapshot = iceberg.table.manageSnapshots()
- val refExists = null != iceberg.table().refs().get(tag)
-
- if (create && replace && !refExists) {
- manageSnapshot.createTag(tag, snapshotId)
- } else if (replace) {
- manageSnapshot.replaceTag(tag, snapshotId)
- } else {
- if (refExists && ifNotExists) {
- return Nil
- }
-
- manageSnapshot.createTag(tag, snapshotId)
- }
-
- if (tagOptions.snapshotRefRetain.nonEmpty) {
- manageSnapshot.setMaxRefAgeMs(tag, tagOptions.snapshotRefRetain.get)
- }
-
- manageSnapshot.commit()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot create tag to non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"Create tag: $tag for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
deleted file mode 100644
index ff8f182009..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class DropBranchExec(
- catalog: TableCatalog,
- ident: Identifier,
- branch: String,
- ifExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val ref = iceberg.table().refs().get(branch)
- if (ref != null || !ifExists) {
- iceberg.table().manageSnapshots().removeBranch(branch).commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop branch on non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropBranch branch: ${branch} for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
deleted file mode 100644
index dee778b474..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions
-import org.apache.iceberg.relocated.com.google.common.collect.Sets
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class DropIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val schema = iceberg.table.schema
- val identifierFieldNames = Sets.newHashSet(schema.identifierFieldNames)
-
- for (name <- fields) {
- Preconditions.checkArgument(schema.findField(name) != null,
- "Cannot complete drop identifier fields operation: field %s not found", name)
- Preconditions.checkArgument(identifierFieldNames.contains(name),
- "Cannot complete drop identifier fields operation: %s is not an identifier field", name)
- identifierFieldNames.remove(name)
- }
-
- iceberg.table.updateSchema()
- .setIdentifierFields(identifierFieldNames)
- .commit();
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop identifier fields in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
deleted file mode 100644
index 9a153f0c00..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.IdentityTransform
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class DropPartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transform: Transform) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val schema = iceberg.table.schema
- transform match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
- // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .commit()
-
- case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transform))
- .commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop partition field in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropPartitionField ${catalog.name}.${ident.quoted} ${transform.describe}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
deleted file mode 100644
index 8df88765a9..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class DropTagExec(
- catalog: TableCatalog,
- ident: Identifier,
- tag: String,
- ifExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val ref = iceberg.table().refs().get(tag)
- if (ref != null || !ifExists) {
- iceberg.table().manageSnapshots().removeTag(tag).commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop tag on non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropTag tag: ${tag} for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala
deleted file mode 100644
index 85bda0b08d..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.connector.catalog.Table
-import org.apache.spark.sql.connector.write.RowLevelOperationTable
-
-/**
- * A class similar to DataSourceV2Implicits in Spark but contains custom implicit helpers.
- */
-object ExtendedDataSourceV2Implicits {
- implicit class TableHelper(table: Table) {
- def asRowLevelOperationTable: RowLevelOperationTable = {
- table match {
- case rowLevelOperationTable: RowLevelOperationTable =>
- rowLevelOperationTable
- case _ =>
- throw new AnalysisException(s"Table ${table.name} is not a row-level operation table")
- }
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
deleted file mode 100644
index 14e59b7f98..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.SparkCatalog
-import org.apache.iceberg.spark.SparkSessionCatalog
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.Strategy
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.Call
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag
-import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.DropBranch
-import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.DropTag
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeRows
-import org.apache.spark.sql.catalyst.plans.logical.NoStatsUnaryNode
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
-import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
-import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-import scala.jdk.CollectionConverters._
-
-case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy with PredicateHelper {
-
- import DataSourceV2Implicits._
-
- override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
- case c @ Call(procedure, args) =>
- val input = buildInternalRow(args)
- CallExec(c.output, procedure, input) :: Nil
-
- case AddPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform, name) =>
- AddPartitionFieldExec(catalog, ident, transform, name) :: Nil
-
- case CreateOrReplaceBranch(
- IcebergCatalogAndIdentifier(catalog, ident), branch, branchOptions, create, replace, ifNotExists) =>
- CreateOrReplaceBranchExec(catalog, ident, branch, branchOptions, create, replace, ifNotExists) :: Nil
-
- case CreateOrReplaceTag(
- IcebergCatalogAndIdentifier(catalog, ident), tag, tagOptions, create, replace, ifNotExists) =>
- CreateOrReplaceTagExec(catalog, ident, tag, tagOptions, create, replace, ifNotExists) :: Nil
-
- case DropBranch(IcebergCatalogAndIdentifier(catalog, ident), branch, ifExists) =>
- DropBranchExec(catalog, ident, branch, ifExists) :: Nil
-
- case DropTag(IcebergCatalogAndIdentifier(catalog, ident), tag, ifExists) =>
- DropTagExec(catalog, ident, tag, ifExists) :: Nil
-
- case DropPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform) =>
- DropPartitionFieldExec(catalog, ident, transform) :: Nil
-
- case ReplacePartitionField(IcebergCatalogAndIdentifier(catalog, ident), transformFrom, transformTo, name) =>
- ReplacePartitionFieldExec(catalog, ident, transformFrom, transformTo, name) :: Nil
-
- case SetIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
- SetIdentifierFieldsExec(catalog, ident, fields) :: Nil
-
- case DropIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
- DropIdentifierFieldsExec(catalog, ident, fields) :: Nil
-
- case SetWriteDistributionAndOrdering(
- IcebergCatalogAndIdentifier(catalog, ident), distributionMode, ordering) =>
- SetWriteDistributionAndOrderingExec(catalog, ident, distributionMode, ordering) :: Nil
-
- case ReplaceIcebergData(_: DataSourceV2Relation, query, r: DataSourceV2Relation, Some(write)) =>
- // refresh the cache using the original relation
- ReplaceDataExec(planLater(query), refreshCache(r), write) :: Nil
-
- case WriteDelta(_: DataSourceV2Relation, query, r: DataSourceV2Relation, projs, Some(write)) =>
- // refresh the cache using the original relation
- WriteDeltaExec(planLater(query), refreshCache(r), projs, write) :: Nil
-
- case MergeRows(isSourceRowPresent, isTargetRowPresent, matchedConditions, matchedOutputs, notMatchedConditions,
- notMatchedOutputs, targetOutput, performCardinalityCheck, emitNotMatchedTargetRows,
- output, child) =>
-
- MergeRowsExec(isSourceRowPresent, isTargetRowPresent, matchedConditions, matchedOutputs, notMatchedConditions,
- notMatchedOutputs, targetOutput, performCardinalityCheck, emitNotMatchedTargetRows,
- output, planLater(child)) :: Nil
-
- case DeleteFromIcebergTable(DataSourceV2ScanRelation(r, _, output, _), condition, None) =>
- // the optimizer has already checked that this delete can be handled using a metadata operation
- val deleteCond = condition.getOrElse(Literal.TrueLiteral)
- val predicates = splitConjunctivePredicates(deleteCond)
- val normalizedPredicates = DataSourceStrategy.normalizeExprs(predicates, output)
- val filters = normalizedPredicates.flatMap { pred =>
- val filter = DataSourceStrategy.translateFilter(pred, supportNestedPredicatePushdown = true)
- if (filter.isEmpty) {
- throw QueryCompilationErrors.cannotTranslateExpressionToSourceFilterError(pred)
- }
- filter
- }.toArray
- DeleteFromTableExec(r.table.asDeletable, filters, refreshCache(r)) :: Nil
-
- case NoStatsUnaryNode(child) =>
- planLater(child) :: Nil
-
- case _ => Nil
- }
-
- private def buildInternalRow(exprs: Seq[Expression]): InternalRow = {
- val values = new Array[Any](exprs.size)
- for (index <- exprs.indices) {
- values(index) = exprs(index).eval()
- }
- new GenericInternalRow(values)
- }
-
- private def refreshCache(r: DataSourceV2Relation)(): Unit = {
- spark.sharedState.cacheManager.recacheByPlan(spark, r)
- }
-
- private object IcebergCatalogAndIdentifier {
- def unapply(identifier: Seq[String]): Option[(TableCatalog, Identifier)] = {
- val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(spark, identifier.asJava)
- catalogAndIdentifier.catalog match {
- case icebergCatalog: SparkCatalog =>
- Some((icebergCatalog, catalogAndIdentifier.identifier))
- case icebergCatalog: SparkSessionCatalog[_] =>
- Some((icebergCatalog, catalogAndIdentifier.identifier))
- case _ =>
- None
- }
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDistributionAndOrderingUtils.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDistributionAndOrderingUtils.scala
deleted file mode 100644
index 8c37b1b759..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDistributionAndOrderingUtils.scala
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils.toCatalyst
-import org.apache.spark.sql.catalyst.expressions.SortOrder
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.RepartitionByExpression
-import org.apache.spark.sql.catalyst.plans.logical.Sort
-import org.apache.spark.sql.connector.distributions.ClusteredDistribution
-import org.apache.spark.sql.connector.distributions.OrderedDistribution
-import org.apache.spark.sql.connector.distributions.UnspecifiedDistribution
-import org.apache.spark.sql.connector.write.RequiresDistributionAndOrdering
-import org.apache.spark.sql.connector.write.Write
-import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.internal.SQLConf
-import scala.collection.compat.immutable.ArraySeq
-
-/**
- * A rule that is inspired by DistributionAndOrderingUtils in Spark but supports Iceberg transforms.
- *
- * Note that similarly to the original rule in Spark, it does not let AQE pick the number of shuffle
- * partitions. See SPARK-34230 for context.
- */
-object ExtendedDistributionAndOrderingUtils {
-
- def prepareQuery(write: Write, query: LogicalPlan, conf: SQLConf): LogicalPlan = write match {
- case write: RequiresDistributionAndOrdering =>
- val numPartitions = write.requiredNumPartitions()
- val distribution = write.requiredDistribution match {
- case d: OrderedDistribution => d.ordering.map(e => toCatalyst(e, query))
- case d: ClusteredDistribution => d.clustering.map(e => toCatalyst(e, query))
- case _: UnspecifiedDistribution => Array.empty[Expression]
- }
-
- val queryWithDistribution = if (distribution.nonEmpty) {
- val finalNumPartitions = if (numPartitions > 0) {
- numPartitions
- } else {
- conf.numShufflePartitions
- }
- // the conversion to catalyst expressions above produces SortOrder expressions
- // for OrderedDistribution and generic expressions for ClusteredDistribution
- // this allows RepartitionByExpression to pick either range or hash partitioning
- RepartitionByExpression(ArraySeq.unsafeWrapArray(distribution), query, finalNumPartitions)
- } else if (numPartitions > 0) {
- throw QueryCompilationErrors.numberOfPartitionsNotAllowedWithUnspecifiedDistributionError()
- } else {
- query
- }
-
- val ordering = write.requiredOrdering.toSeq
- .map(e => toCatalyst(e, query))
- .asInstanceOf[Seq[SortOrder]]
-
- val queryWithDistributionAndOrdering = if (ordering.nonEmpty) {
- Sort(ordering, global = false, queryWithDistribution)
- } else {
- queryWithDistribution
- }
-
- queryWithDistributionAndOrdering
-
- case _ =>
- query
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala
deleted file mode 100644
index 3e60d2494b..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import java.util.UUID
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.plans.logical.AppendData
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.OverwriteByExpression
-import org.apache.spark.sql.catalyst.plans.logical.OverwritePartitionsDynamic
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.utils.PlanUtils.isIcebergRelation
-import org.apache.spark.sql.connector.catalog.Table
-import org.apache.spark.sql.connector.iceberg.write.DeltaWriteBuilder
-import org.apache.spark.sql.connector.write.ExtendedLogicalWriteInfoImpl
-import org.apache.spark.sql.connector.write.SupportsDynamicOverwrite
-import org.apache.spark.sql.connector.write.SupportsOverwrite
-import org.apache.spark.sql.connector.write.SupportsTruncate
-import org.apache.spark.sql.connector.write.WriteBuilder
-import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.errors.QueryExecutionErrors
-import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-import org.apache.spark.sql.sources.AlwaysTrue
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types.StructType
-
-/**
- * A rule that is inspired by V2Writes in Spark but supports Iceberg transforms.
- */
-object ExtendedV2Writes extends Rule[LogicalPlan] with PredicateHelper {
-
- import DataSourceV2Implicits._
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
- case a @ AppendData(r: DataSourceV2Relation, query, options, _, None) if isIcebergRelation(r) =>
- val writeBuilder = newWriteBuilder(r.table, query.schema, options)
- val write = writeBuilder.build()
- val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
- a.copy(write = Some(write), query = newQuery)
-
- case o @ OverwriteByExpression(r: DataSourceV2Relation, deleteExpr, query, options, _, None)
- if isIcebergRelation(r) =>
- // fail if any filter cannot be converted. correctness depends on removing all matching data.
- val filters = splitConjunctivePredicates(deleteExpr).flatMap { pred =>
- val filter = DataSourceStrategy.translateFilter(pred, supportNestedPredicatePushdown = true)
- if (filter.isEmpty) {
- throw QueryCompilationErrors.cannotTranslateExpressionToSourceFilterError(pred)
- }
- filter
- }.toArray
-
- val table = r.table
- val writeBuilder = newWriteBuilder(table, query.schema, options)
- val write = writeBuilder match {
- case builder: SupportsTruncate if isTruncate(filters) =>
- builder.truncate().build()
- case builder: SupportsOverwrite =>
- builder.overwrite(filters).build()
- case _ =>
- throw QueryExecutionErrors.overwriteTableByUnsupportedExpressionError(table)
- }
-
- val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
- o.copy(write = Some(write), query = newQuery)
-
- case o @ OverwritePartitionsDynamic(r: DataSourceV2Relation, query, options, _, None)
- if isIcebergRelation(r) =>
- val table = r.table
- val writeBuilder = newWriteBuilder(table, query.schema, options)
- val write = writeBuilder match {
- case builder: SupportsDynamicOverwrite =>
- builder.overwriteDynamicPartitions().build()
- case _ =>
- throw QueryExecutionErrors.dynamicPartitionOverwriteUnsupportedByTableError(table)
- }
- val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
- o.copy(write = Some(write), query = newQuery)
-
- case rd @ ReplaceIcebergData(r: DataSourceV2Relation, query, _, None) =>
- val rowSchema = StructType.fromAttributes(rd.dataInput)
- val writeBuilder = newWriteBuilder(r.table, rowSchema, Map.empty)
- val write = writeBuilder.build()
- val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(write, query, conf)
- rd.copy(write = Some(write), query = Project(rd.dataInput, newQuery))
-
- case wd @ WriteDelta(r: DataSourceV2Relation, query, _, projections, None) =>
- val rowSchema = projections.rowProjection.map(_.schema).orNull
- val rowIdSchema = projections.rowIdProjection.schema
- val metadataSchema = projections.metadataProjection.map(_.schema).orNull
- val writeBuilder = newWriteBuilder(r.table, rowSchema, Map.empty, rowIdSchema, metadataSchema)
- writeBuilder match {
- case builder: DeltaWriteBuilder =>
- val deltaWrite = builder.build()
- val newQuery = ExtendedDistributionAndOrderingUtils.prepareQuery(deltaWrite, query, conf)
- wd.copy(write = Some(deltaWrite), query = newQuery)
- case other =>
- throw new AnalysisException(s"$other is not DeltaWriteBuilder")
- }
- }
-
- private def isTruncate(filters: Array[Filter]): Boolean = {
- filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
- }
-
- private def newWriteBuilder(
- table: Table,
- rowSchema: StructType,
- writeOptions: Map[String, String],
- rowIdSchema: StructType = null,
- metadataSchema: StructType = null): WriteBuilder = {
- val info = ExtendedLogicalWriteInfoImpl(
- queryId = UUID.randomUUID().toString,
- rowSchema,
- writeOptions.asOptions,
- rowIdSchema,
- metadataSchema)
- table.asWritable.newWriteBuilder(info)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala
deleted file mode 100644
index 0f9d0fa7ac..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.SparkException
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-import org.apache.spark.sql.catalyst.expressions.BasePredicate
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
-import org.apache.spark.sql.catalyst.expressions.codegen.GeneratePredicate
-import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.UnaryExecNode
-import org.roaringbitmap.longlong.Roaring64Bitmap
-
-case class MergeRowsExec(
- isSourceRowPresent: Expression,
- isTargetRowPresent: Expression,
- matchedConditions: Seq[Expression],
- matchedOutputs: Seq[Seq[Expression]],
- notMatchedConditions: Seq[Expression],
- notMatchedOutputs: Seq[Seq[Expression]],
- targetOutput: Seq[Expression],
- performCardinalityCheck: Boolean,
- emitNotMatchedTargetRows: Boolean,
- output: Seq[Attribute],
- child: SparkPlan) extends UnaryExecNode {
-
- private final val ROW_ID = "__row_id"
-
- @transient override lazy val producedAttributes: AttributeSet = {
- AttributeSet(output.filterNot(attr => inputSet.contains(attr)))
- }
-
- @transient override lazy val references: AttributeSet = child.outputSet
-
- override def simpleString(maxFields: Int): String = {
- s"MergeRowsExec${truncatedString(output, "[", ", ", "]", maxFields)}"
- }
-
- override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = {
- copy(child = newChild)
- }
-
- protected override def doExecute(): RDD[InternalRow] = {
- child.execute().mapPartitions(processPartition)
- }
-
- private def createProjection(exprs: Seq[Expression], attrs: Seq[Attribute]): UnsafeProjection = {
- UnsafeProjection.create(exprs, attrs)
- }
-
- private def createPredicate(expr: Expression, attrs: Seq[Attribute]): BasePredicate = {
- GeneratePredicate.generate(expr, attrs)
- }
-
- private def applyProjection(
- actions: Seq[(BasePredicate, Option[UnsafeProjection])],
- inputRow: InternalRow): InternalRow = {
-
- // find the first action where the predicate evaluates to true
- // if there are overlapping conditions in actions, use the first matching action
- // in the example below, when id = 5, both actions match but the first one is applied
- // WHEN MATCHED AND id > 1 AND id < 10 UPDATE *
- // WHEN MATCHED AND id = 5 OR id = 21 DELETE
-
- val pair = actions.find {
- case (predicate, _) => predicate.eval(inputRow)
- }
-
- // apply the projection to produce an output row, or return null to suppress this row
- pair match {
- case Some((_, Some(projection))) =>
- projection.apply(inputRow)
- case _ =>
- null
- }
- }
-
- private def processPartition(rowIterator: Iterator[InternalRow]): Iterator[InternalRow] = {
- val inputAttrs = child.output
-
- val isSourceRowPresentPred = createPredicate(isSourceRowPresent, inputAttrs)
- val isTargetRowPresentPred = createPredicate(isTargetRowPresent, inputAttrs)
-
- val matchedPreds = matchedConditions.map(createPredicate(_, inputAttrs))
- val matchedProjs = matchedOutputs.map {
- case output if output.nonEmpty => Some(createProjection(output, inputAttrs))
- case _ => None
- }
- val matchedPairs = matchedPreds zip matchedProjs
-
- val notMatchedPreds = notMatchedConditions.map(createPredicate(_, inputAttrs))
- val notMatchedProjs = notMatchedOutputs.map {
- case output if output.nonEmpty => Some(createProjection(output, inputAttrs))
- case _ => None
- }
- val nonMatchedPairs = notMatchedPreds zip notMatchedProjs
-
- val projectTargetCols = createProjection(targetOutput, inputAttrs)
-
- // This method is responsible for processing a input row to emit the resultant row with an
- // additional column that indicates whether the row is going to be included in the final
- // output of merge or not.
- // 1. Found a target row for which there is no corresponding source row (join condition not met)
- // - Only project the target columns if we need to output unchanged rows
- // 2. Found a source row for which there is no corresponding target row (join condition not met)
- // - Apply the not matched actions (i.e INSERT actions) if non match conditions are met.
- // 3. Found a source row for which there is a corresponding target row (join condition met)
- // - Apply the matched actions (i.e DELETE or UPDATE actions) if match conditions are met.
- def processRow(inputRow: InternalRow): InternalRow = {
- if (emitNotMatchedTargetRows && !isSourceRowPresentPred.eval(inputRow)) {
- projectTargetCols.apply(inputRow)
- } else if (!isTargetRowPresentPred.eval(inputRow)) {
- applyProjection(nonMatchedPairs, inputRow)
- } else {
- applyProjection(matchedPairs, inputRow)
- }
- }
-
- val matchedRowIds = new Roaring64Bitmap()
-
- def processRowWithCardinalityCheck(rowIdOrdinal: Int)(inputRow: InternalRow): InternalRow = {
- val isSourceRowPresent = isSourceRowPresentPred.eval(inputRow)
- val isTargetRowPresent = isTargetRowPresentPred.eval(inputRow)
-
- if (isSourceRowPresent && isTargetRowPresent) {
- val currentRowId = inputRow.getLong(rowIdOrdinal)
- if (matchedRowIds.contains(currentRowId)) {
- throw new SparkException(
- "The ON search condition of the MERGE statement matched a single row from " +
- "the target table with multiple rows of the source table. This could result " +
- "in the target row being operated on more than once with an update or delete " +
- "operation and is not allowed.")
- }
- matchedRowIds.add(currentRowId)
- }
-
- if (emitNotMatchedTargetRows && !isSourceRowPresent) {
- projectTargetCols.apply(inputRow)
- } else if (!isTargetRowPresent) {
- applyProjection(nonMatchedPairs, inputRow)
- } else {
- applyProjection(matchedPairs, inputRow)
- }
- }
-
- val processFunc: InternalRow => InternalRow = if (performCardinalityCheck) {
- val rowIdOrdinal = child.output.indexWhere(attr => conf.resolver(attr.name, ROW_ID))
- assert(rowIdOrdinal != -1, "Cannot find row ID attr")
- processRowWithCardinalityCheck(rowIdOrdinal)
- } else {
- processRow
- }
-
- rowIterator
- .map(processFunc)
- .filter(row => row != null)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromIcebergTable.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromIcebergTable.scala
deleted file mode 100644
index e81a567eb2..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromIcebergTable.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.SupportsDelete
-import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-import org.apache.spark.sql.sources
-import org.slf4j.LoggerFactory
-
-/**
- * Checks whether a metadata delete is possible and nullifies the rewrite plan if the source can
- * handle this delete without executing the rewrite plan.
- *
- * Note this rule must be run after expression optimization.
- */
-object OptimizeMetadataOnlyDeleteFromIcebergTable extends Rule[LogicalPlan] with PredicateHelper {
-
- val logger = LoggerFactory.getLogger(OptimizeMetadataOnlyDeleteFromIcebergTable.getClass)
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan transform {
- case d @ DeleteFromIcebergTable(relation: DataSourceV2Relation, cond, Some(_)) =>
- val deleteCond = cond.getOrElse(Literal.TrueLiteral)
- relation.table match {
- case table: SupportsDelete if !SubqueryExpression.hasSubquery(deleteCond) =>
- val predicates = splitConjunctivePredicates(deleteCond)
- val normalizedPredicates = DataSourceStrategy.normalizeExprs(predicates, relation.output)
- val dataSourceFilters = toDataSourceFilters(normalizedPredicates)
- val allPredicatesTranslated = normalizedPredicates.size == dataSourceFilters.length
- if (allPredicatesTranslated && table.canDeleteWhere(dataSourceFilters)) {
- logger.info(s"Optimizing delete expression: ${dataSourceFilters.mkString(",")} as metadata delete")
- d.copy(rewritePlan = None)
- } else {
- d
- }
- case _ =>
- d
- }
- }
-
- protected def toDataSourceFilters(predicates: Seq[Expression]): Array[sources.Filter] = {
- predicates.flatMap { p =>
- val filter = DataSourceStrategy.translateFilter(p, supportNestedPredicatePushdown = true)
- if (filter.isEmpty) {
- logWarning(s"Cannot translate expression to source filter: $p")
- }
- filter
- }.toArray
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala
deleted file mode 100644
index 26c652469a..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-import org.apache.spark.sql.connector.write.Write
-import org.apache.spark.sql.execution.SparkPlan
-
-/**
- * Physical plan node to replace data in existing tables.
- */
-case class ReplaceDataExec(
- query: SparkPlan,
- refreshCache: () => Unit,
- write: Write) extends V2ExistingTableWriteExec {
-
- override lazy val references: AttributeSet = query.outputSet
- override lazy val stringArgs: Iterator[Any] = Iterator(query, write)
-
- override protected def withNewChildInternal(newChild: SparkPlan): ReplaceDataExec = {
- copy(query = newChild)
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
deleted file mode 100644
index fcae0a5def..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.IdentityTransform
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class ReplacePartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transformFrom: Transform,
- transformTo: Transform,
- name: Option[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val schema = iceberg.table.schema
- transformFrom match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
- // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
-
- case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transformFrom))
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot replace partition field in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"ReplacePartitionField ${catalog.name}.${ident.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala
deleted file mode 100644
index 414d4c0ec3..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.RowLevelCommand
-import org.apache.spark.sql.catalyst.rules.Rule
-
-/**
- * Replaces operations such as DELETE and MERGE with the corresponding rewrite plans.
- */
-object ReplaceRewrittenRowLevelCommand extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
- case c: RowLevelCommand if c.rewritePlan.isDefined =>
- c.rewritePlan.get
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
deleted file mode 100644
index 96f6843feb..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.expressions.And
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.planning.RewrittenRowLevelCommand
-import org.apache.spark.sql.catalyst.planning.ScanOperation
-import org.apache.spark.sql.catalyst.plans.logical.Join
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.NoStatsUnaryNode
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.RowLevelCommand
-import org.apache.spark.sql.catalyst.plans.logical.WriteDelta
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.util.CharVarcharUtils
-import org.apache.spark.sql.connector.expressions.filter.Predicate
-import org.apache.spark.sql.connector.read.ScanBuilder
-import org.apache.spark.sql.execution.datasources.DataSourceStrategy
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types.StructType
-
-object RowLevelCommandScanRelationPushDown extends Rule[LogicalPlan] with PredicateHelper {
- import ExtendedDataSourceV2Implicits._
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
- // use native Spark planning for delta-based plans
- // unlike other commands, these plans have filters that can be pushed down directly
- case RewrittenRowLevelCommand(command, _: DataSourceV2Relation, rewritePlan: WriteDelta) =>
- val newRewritePlan = V2ScanRelationPushDown.apply(rewritePlan)
- command.withNewRewritePlan(newRewritePlan)
-
- // group-based MERGE operations are rewritten as joins and may be planned in a special way
- // the join condition is the MERGE condition and can be pushed into the source
- // this allows us to remove completely pushed down predicates from the join condition
- case UnplannedGroupBasedMergeOperation(command, rd: ReplaceIcebergData,
- join @ Join(_, _, _, Some(joinCond), _), relation: DataSourceV2Relation) =>
-
- val table = relation.table.asRowLevelOperationTable
- val scanBuilder = table.newScanBuilder(relation.options)
-
- val (pushedFilters, newJoinCond) = pushMergeFilters(joinCond, relation, scanBuilder)
- val pushedFiltersStr = if (pushedFilters.isLeft) {
- pushedFilters.left.get.mkString(", ")
- } else {
- pushedFilters.right.get.mkString(", ")
- }
-
- val (scan, output) = PushDownUtils.pruneColumns(scanBuilder, relation, relation.output, Nil)
-
- logInfo(
- s"""
- |Pushing MERGE operators to ${relation.name}
- |Pushed filters: $pushedFiltersStr
- |Original JOIN condition: $joinCond
- |New JOIN condition: $newJoinCond
- |Output: ${output.mkString(", ")}
- """.stripMargin)
-
- val newRewritePlan = rd transformDown {
- case j: Join if j eq join =>
- j.copy(condition = newJoinCond)
- case r: DataSourceV2Relation if r.table eq table =>
- DataSourceV2ScanRelation(r, scan, PushDownUtils.toOutputAttrs(scan.readSchema(), r))
- }
-
- command.withNewRewritePlan(newRewritePlan)
-
- // push down the filter from the command condition instead of the filter in the rewrite plan,
- // which may be negated for copy-on-write DELETE and UPDATE operations
- case RewrittenRowLevelCommand(command, relation: DataSourceV2Relation, rewritePlan) =>
- val table = relation.table.asRowLevelOperationTable
- val scanBuilder = table.newScanBuilder(relation.options)
-
- val (pushedFilters, remainingFilters) = command.condition match {
- case Some(cond) => pushFilters(cond, scanBuilder, relation.output)
- case None => (Left(Nil), Nil)
- }
-
- val pushedFiltersStr = if (pushedFilters.isLeft) {
- pushedFilters.left.get.mkString(", ")
- } else {
- pushedFilters.right.get.mkString(", ")
- }
-
- val (scan, output) = PushDownUtils.pruneColumns(scanBuilder, relation, relation.output, Nil)
-
- logInfo(
- s"""
- |Pushing operators to ${relation.name}
- |Pushed filters: $pushedFiltersStr
- |Filters that were not pushed: ${remainingFilters.mkString(",")}
- |Output: ${output.mkString(", ")}
- """.stripMargin)
-
- // replace DataSourceV2Relation with DataSourceV2ScanRelation for the row operation table
- // there may be multiple read relations for UPDATEs that rely on the UNION approach
- val newRewritePlan = rewritePlan transform {
- case r: DataSourceV2Relation if r.table eq table =>
- DataSourceV2ScanRelation(r, scan, toOutputAttrs(scan.readSchema(), r))
- }
-
- command.withNewRewritePlan(newRewritePlan)
- }
-
- private def pushFilters(
- cond: Expression,
- scanBuilder: ScanBuilder,
- tableAttrs: Seq[AttributeReference]): (Either[Seq[Filter], Seq[Predicate]], Seq[Expression]) = {
-
- val tableAttrSet = AttributeSet(tableAttrs)
- val filters = splitConjunctivePredicates(cond).filter(_.references.subsetOf(tableAttrSet))
- val normalizedFilters = DataSourceStrategy.normalizeExprs(filters, tableAttrs)
- val (_, normalizedFiltersWithoutSubquery) =
- normalizedFilters.partition(SubqueryExpression.hasSubquery)
-
- PushDownUtils.pushFilters(scanBuilder, normalizedFiltersWithoutSubquery)
- }
-
- // splits the join condition into predicates and tries to push down each predicate into the scan
- // completely pushed down predicates are removed from the join condition
- // joinCond can't have subqueries as it is validated by the rule that rewrites MERGE as a join
- private def pushMergeFilters(
- joinCond: Expression,
- relation: DataSourceV2Relation,
- scanBuilder: ScanBuilder): (Either[Seq[Filter], Seq[Predicate]], Option[Expression]) = {
-
- val (tableFilters, commonFilters) =
- splitConjunctivePredicates(joinCond).partition(_.references.subsetOf(relation.outputSet))
- val normalizedTableFilters = DataSourceStrategy.normalizeExprs(tableFilters, relation.output)
- val (pushedFilters, postScanFilters) =
- PushDownUtils.pushFilters(scanBuilder, normalizedTableFilters)
- val newJoinCond = (commonFilters ++ postScanFilters).reduceLeftOption(And)
-
- (pushedFilters, newJoinCond)
- }
-
- private def toOutputAttrs(
- schema: StructType,
- relation: DataSourceV2Relation): Seq[AttributeReference] = {
- val nameToAttr = relation.output.map(_.name).zip(relation.output).toMap
- val cleaned = CharVarcharUtils.replaceCharVarcharWithStringInSchema(schema)
- cleaned.toAttributes.map {
- // keep the attribute id during transformation
- a => a.withExprId(nameToAttr(a.name).exprId)
- }
- }
-}
-
-object UnplannedGroupBasedMergeOperation {
- type ReturnType = (RowLevelCommand, ReplaceIcebergData, Join, DataSourceV2Relation)
-
- def unapply(plan: LogicalPlan): Option[ReturnType] = plan match {
- case m @ MergeIntoIcebergTable(_, _, _, _, _, Some(rewritePlan)) =>
- rewritePlan match {
- case rd @ ReplaceIcebergData(DataSourceV2Relation(table, _, _, _, _), query, _, _) =>
- val joinsAndRelations = query.collect {
- case j @ Join(
- NoStatsUnaryNode(ScanOperation(_, filters, r: DataSourceV2Relation)), _, _, _, _)
- if filters.isEmpty && r.table.eq(table) =>
- j -> r
- }
-
- joinsAndRelations match {
- case Seq((join, relation)) =>
- Some(m, rd, join, relation)
- case _ =>
- None
- }
-
- case _ =>
- None
- }
-
- case _ =>
- None
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
deleted file mode 100644
index b50550ad38..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import scala.jdk.CollectionConverters._
-
-case class SetIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- iceberg.table.updateSchema()
- .setIdentifierFields(fields.asJava)
- .commit();
- case table =>
- throw new UnsupportedOperationException(s"Cannot set identifier fields in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"SetIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
deleted file mode 100644
index feecc02350..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.DistributionMode
-import org.apache.iceberg.NullOrder
-import org.apache.iceberg.SortDirection
-import org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE
-import org.apache.iceberg.expressions.Term
-import org.apache.iceberg.spark.SparkUtil
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class SetWriteDistributionAndOrderingExec(
- catalog: TableCatalog,
- ident: Identifier,
- distributionMode: DistributionMode,
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafV2CommandExec {
-
- import CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val txn = iceberg.table.newTransaction()
-
- val orderBuilder = txn.replaceSortOrder().caseSensitive(SparkUtil.caseSensitive(session))
- sortOrder.foreach {
- case (term, SortDirection.ASC, nullOrder) =>
- orderBuilder.asc(term, nullOrder)
- case (term, SortDirection.DESC, nullOrder) =>
- orderBuilder.desc(term, nullOrder)
- }
- orderBuilder.commit()
-
- txn.updateProperties()
- .set(WRITE_DISTRIBUTION_MODE, distributionMode.modeName())
- .commit()
-
- txn.commitTransaction()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot set write order of non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- val tableIdent = s"${catalog.name}.${ident.quoted}"
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
- s"SetWriteDistributionAndOrdering $tableIdent $distributionMode $order"
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteDeltaExec.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteDeltaExec.scala
deleted file mode 100644
index fa4e4f6483..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteDeltaExec.scala
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.SparkEnv
-import org.apache.spark.SparkException
-import org.apache.spark.TaskContext
-import org.apache.spark.internal.Logging
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.AttributeSet
-import org.apache.spark.sql.catalyst.util.RowDeltaUtils._
-import org.apache.spark.sql.catalyst.util.WriteDeltaProjections
-import org.apache.spark.sql.connector.iceberg.write.DeltaWrite
-import org.apache.spark.sql.connector.iceberg.write.DeltaWriter
-import org.apache.spark.sql.connector.write.BatchWrite
-import org.apache.spark.sql.connector.write.DataWriter
-import org.apache.spark.sql.connector.write.DataWriterFactory
-import org.apache.spark.sql.connector.write.PhysicalWriteInfoImpl
-import org.apache.spark.sql.connector.write.WriterCommitMessage
-import org.apache.spark.sql.errors.QueryExecutionErrors
-import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.metric.CustomMetrics
-import org.apache.spark.sql.execution.metric.SQLMetric
-import org.apache.spark.util.LongAccumulator
-import org.apache.spark.util.Utils
-import scala.collection.compat.immutable.ArraySeq
-import scala.util.control.NonFatal
-
-/**
- * Physical plan node to write a delta of rows to an existing table.
- */
-case class WriteDeltaExec(
- query: SparkPlan,
- refreshCache: () => Unit,
- projections: WriteDeltaProjections,
- write: DeltaWrite) extends ExtendedV2ExistingTableWriteExec[DeltaWriter[InternalRow]] {
-
- override lazy val references: AttributeSet = query.outputSet
- override lazy val stringArgs: Iterator[Any] = Iterator(query, write)
-
- override lazy val writingTask: WritingSparkTask[DeltaWriter[InternalRow]] = {
- DeltaWithMetadataWritingSparkTask(projections)
- }
-
- override protected def withNewChildInternal(newChild: SparkPlan): WriteDeltaExec = {
- copy(query = newChild)
- }
-}
-
-// a trait similar to V2ExistingTableWriteExec but supports custom write tasks
-trait ExtendedV2ExistingTableWriteExec[W <: DataWriter[InternalRow]] extends V2ExistingTableWriteExec {
- def writingTask: WritingSparkTask[W]
-
- protected override def writeWithV2(batchWrite: BatchWrite): Seq[InternalRow] = {
- val rdd: RDD[InternalRow] = {
- val tempRdd = query.execute()
- // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single
- // partition rdd to make sure we at least set up one write task to write the metadata.
- if (tempRdd.partitions.length == 0) {
- sparkContext.parallelize(Seq.empty[InternalRow], 1)
- } else {
- tempRdd
- }
- }
- // introduce a local var to avoid serializing the whole class
- val task = writingTask
- val writerFactory = batchWrite.createBatchWriterFactory(
- PhysicalWriteInfoImpl(rdd.getNumPartitions))
- val useCommitCoordinator = batchWrite.useCommitCoordinator
- val messages = new Array[WriterCommitMessage](rdd.partitions.length)
- val totalNumRowsAccumulator = new LongAccumulator()
-
- logInfo(s"Start processing data source write support: $batchWrite. " +
- s"The input RDD has ${messages.length} partitions.")
-
- // Avoid object not serializable issue.
- val writeMetrics: Map[String, SQLMetric] = customMetrics
-
- try {
- sparkContext.runJob(
- rdd,
- (context: TaskContext, iter: Iterator[InternalRow]) =>
- task.run(writerFactory, context, iter, useCommitCoordinator, writeMetrics),
- rdd.partitions.indices,
- (index, result: DataWritingSparkTaskResult) => {
- val commitMessage = result.writerCommitMessage
- messages(index) = commitMessage
- totalNumRowsAccumulator.add(result.numRows)
- batchWrite.onDataWriterCommit(commitMessage)
- }
- )
-
- logInfo(s"Data source write support $batchWrite is committing.")
- batchWrite.commit(messages)
- logInfo(s"Data source write support $batchWrite committed.")
- commitProgress = Some(StreamWriterCommitProgress(totalNumRowsAccumulator.value))
- } catch {
- case cause: Throwable =>
- logError(s"Data source write support $batchWrite is aborting.")
- try {
- batchWrite.abort(messages)
- } catch {
- case t: Throwable =>
- logError(s"Data source write support $batchWrite failed to abort.")
- cause.addSuppressed(t)
- throw QueryExecutionErrors.writingJobFailedError(cause)
- }
- logError(s"Data source write support $batchWrite aborted.")
- cause match {
- // Only wrap non fatal exceptions.
- case NonFatal(e) => throw QueryExecutionErrors.writingJobAbortedError(e)
- case _ => throw cause
- }
- }
-
- Nil
- }
-}
-
-trait WritingSparkTask[W <: DataWriter[InternalRow]] extends Logging with Serializable {
-
- protected def writeFunc(writer: W, row: InternalRow): Unit
-
- def run(
- writerFactory: DataWriterFactory,
- context: TaskContext,
- iter: Iterator[InternalRow],
- useCommitCoordinator: Boolean,
- customMetrics: Map[String, SQLMetric]): DataWritingSparkTaskResult = {
- val stageId = context.stageId()
- val stageAttempt = context.stageAttemptNumber()
- val partId = context.partitionId()
- val taskId = context.taskAttemptId()
- val attemptId = context.attemptNumber()
- val dataWriter = writerFactory.createWriter(partId, taskId).asInstanceOf[W]
-
- var count = 0L
- // write the data and commit this writer.
- Utils.tryWithSafeFinallyAndFailureCallbacks(block = {
- while (iter.hasNext) {
- if (count % CustomMetrics.NUM_ROWS_PER_UPDATE == 0) {
- CustomMetrics.updateMetrics(ArraySeq.unsafeWrapArray(dataWriter.currentMetricsValues), customMetrics)
- }
-
- // Count is here.
- count += 1
- writeFunc(dataWriter, iter.next())
- }
-
- CustomMetrics.updateMetrics(ArraySeq.unsafeWrapArray(dataWriter.currentMetricsValues), customMetrics)
-
- val msg = if (useCommitCoordinator) {
- val coordinator = SparkEnv.get.outputCommitCoordinator
- val commitAuthorized = coordinator.canCommit(stageId, stageAttempt, partId, attemptId)
- if (commitAuthorized) {
- logInfo(s"Commit authorized for partition $partId (task $taskId, attempt $attemptId, " +
- s"stage $stageId.$stageAttempt)")
- dataWriter.commit()
- } else {
- val commitDeniedException = QueryExecutionErrors.commitDeniedError(
- partId, taskId, attemptId, stageId, stageAttempt)
- logInfo(commitDeniedException.getMessage)
- // throwing CommitDeniedException will trigger the catch block for abort
- throw commitDeniedException
- }
-
- } else {
- logInfo(s"Writer for partition ${context.partitionId()} is committing.")
- dataWriter.commit()
- }
-
- logInfo(s"Committed partition $partId (task $taskId, attempt $attemptId, " +
- s"stage $stageId.$stageAttempt)")
-
- DataWritingSparkTaskResult(count, msg)
-
- })(catchBlock = {
- // If there is an error, abort this writer
- logError(s"Aborting commit for partition $partId (task $taskId, attempt $attemptId, " +
- s"stage $stageId.$stageAttempt)")
- dataWriter.abort()
- logError(s"Aborted commit for partition $partId (task $taskId, attempt $attemptId, " +
- s"stage $stageId.$stageAttempt)")
- }, finallyBlock = {
- dataWriter.close()
- })
- }
-}
-
-case class DeltaWithMetadataWritingSparkTask(
- projs: WriteDeltaProjections) extends WritingSparkTask[DeltaWriter[InternalRow]] {
-
- private lazy val rowProjection = projs.rowProjection.orNull
- private lazy val rowIdProjection = projs.rowIdProjection
- private lazy val metadataProjection = projs.metadataProjection.orNull
-
- override protected def writeFunc(writer: DeltaWriter[InternalRow], row: InternalRow): Unit = {
- val operation = row.getInt(0)
-
- operation match {
- case DELETE_OPERATION =>
- rowIdProjection.project(row)
- metadataProjection.project(row)
- writer.delete(metadataProjection, rowIdProjection)
-
- case UPDATE_OPERATION =>
- rowProjection.project(row)
- rowIdProjection.project(row)
- metadataProjection.project(row)
- writer.update(metadataProjection, rowIdProjection, rowProjection)
-
- case INSERT_OPERATION =>
- rowProjection.project(row)
- writer.insert(rowProjection)
-
- case other =>
- throw new SparkException(s"Unexpected operation ID: $other")
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala b/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
deleted file mode 100644
index d44e2d7a30..0000000000
--- a/spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.dynamicpruning
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.expressions.And
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.AttributeMap
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.expressions.DynamicPruningSubquery
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.ExtendedV2ExpressionUtils
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.planning.RewrittenRowLevelCommand
-import org.apache.spark.sql.catalyst.plans.LeftSemi
-import org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.plans.logical.Join
-import org.apache.spark.sql.catalyst.plans.logical.JoinHint
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceIcebergData
-import org.apache.spark.sql.catalyst.plans.logical.RowLevelCommand
-import org.apache.spark.sql.catalyst.plans.logical.Sort
-import org.apache.spark.sql.catalyst.plans.logical.Subquery
-import org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION
-import org.apache.spark.sql.catalyst.trees.TreePattern.SORT
-import org.apache.spark.sql.connector.read.SupportsRuntimeFiltering
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
-import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Implicits
-import scala.collection.compat.immutable.ArraySeq
-
-/**
- * A rule that adds a runtime filter for row-level commands.
- *
- * Note that only group-based rewrite plans (i.e. ReplaceData) are taken into account.
- * Row-based rewrite plans are subject to usual runtime filtering.
- */
-case class RowLevelCommandDynamicPruning(spark: SparkSession) extends Rule[LogicalPlan] with PredicateHelper {
-
- import ExtendedDataSourceV2Implicits._
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
- // apply special dynamic filtering only for plans that don't support deltas
- case RewrittenRowLevelCommand(
- command: RowLevelCommand,
- DataSourceV2ScanRelation(_, scan: SupportsRuntimeFiltering, _, _),
- rewritePlan: ReplaceIcebergData) if conf.dynamicPartitionPruningEnabled && isCandidate(command) =>
-
- // use reference equality to find exactly the required scan relations
- val newRewritePlan = rewritePlan transformUp {
- case r: DataSourceV2ScanRelation if r.scan eq scan =>
- // use the original table instance that was loaded for this row-level operation
- // in order to leverage a regular batch scan in the group filter query
- val originalTable = r.relation.table.asRowLevelOperationTable.table
- val relation = r.relation.copy(table = originalTable)
- val matchingRowsPlan = buildMatchingRowsPlan(relation, command)
-
- val filterAttrs = ArraySeq.unsafeWrapArray(scan.filterAttributes)
- val buildKeys = ExtendedV2ExpressionUtils.resolveRefs[Attribute](filterAttrs, matchingRowsPlan)
- val pruningKeys = ExtendedV2ExpressionUtils.resolveRefs[Attribute](filterAttrs, r)
- val dynamicPruningCond = buildDynamicPruningCond(matchingRowsPlan, buildKeys, pruningKeys)
-
- Filter(dynamicPruningCond, r)
- }
-
- // always optimize dynamic filtering subqueries for row-level commands as it is important
- // to rewrite introduced predicates as joins because Spark recently stopped optimizing
- // dynamic subqueries to facilitate broadcast reuse
- command.withNewRewritePlan(optimizeSubquery(newRewritePlan))
- }
-
- private def isCandidate(command: RowLevelCommand): Boolean = command.condition match {
- case Some(cond) if cond != Literal.TrueLiteral => true
- case _ => false
- }
-
- private def buildMatchingRowsPlan(
- relation: DataSourceV2Relation,
- command: RowLevelCommand): LogicalPlan = {
-
- // construct a filtering plan with the original scan relation
- val matchingRowsPlan = command match {
- case d: DeleteFromIcebergTable =>
- Filter(d.condition.get, relation)
-
- case u: UpdateIcebergTable =>
- // UPDATEs with subqueries are rewritten using a UNION with two identical scan relations
- // the analyzer clones of them and assigns fresh expr IDs so that attributes don't collide
- // this rule assigns dynamic filters to both scan relations based on the update condition
- // the condition always refers to the original expr IDs and must be transformed
- // see RewriteUpdateTable for more details
- val attrMap = buildAttrMap(u.table.output, relation.output)
- val transformedCond = u.condition.get transform {
- case attr: AttributeReference if attrMap.contains(attr) => attrMap(attr)
- }
- Filter(transformedCond, relation)
-
- case m: MergeIntoIcebergTable =>
- Join(relation, m.sourceTable, LeftSemi, Some(m.mergeCondition), JoinHint.NONE)
- }
-
- // clone the original relation in the filtering plan and assign new expr IDs to avoid conflicts
- matchingRowsPlan transformUpWithNewOutput {
- case r: DataSourceV2Relation if r eq relation =>
- val oldOutput = r.output
- val newOutput = oldOutput.map(_.newInstance())
- r.copy(output = newOutput) -> oldOutput.zip(newOutput)
- }
- }
-
- private def buildDynamicPruningCond(
- matchingRowsPlan: LogicalPlan,
- buildKeys: Seq[Attribute],
- pruningKeys: Seq[Attribute]): Expression = {
-
- val buildQuery = Project(buildKeys, matchingRowsPlan)
- val dynamicPruningSubqueries = pruningKeys.zipWithIndex.map { case (key, index) =>
- DynamicPruningSubquery(key, buildQuery, buildKeys, index, onlyInBroadcast = false)
- }
- dynamicPruningSubqueries.reduce(And)
- }
-
- private def buildAttrMap(
- tableAttrs: Seq[Attribute],
- scanAttrs: Seq[Attribute]): AttributeMap[Attribute] = {
-
- val resolver = conf.resolver
- val attrMapping = tableAttrs.flatMap { tableAttr =>
- scanAttrs
- .find(scanAttr => resolver(scanAttr.name, tableAttr.name))
- .map(scanAttr => tableAttr -> scanAttr)
- }
- AttributeMap(attrMapping)
- }
-
- // borrowed from OptimizeSubqueries in Spark
- private def optimizeSubquery(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
- _.containsPattern(PLAN_EXPRESSION)) {
- case s: SubqueryExpression =>
- val Subquery(newPlan, _) = spark.sessionState.optimizer.execute(Subquery.fromExpression(s))
- // At this point we have an optimized subquery plan that we are going to attach
- // to this subquery expression. Here we can safely remove any top level sort
- // in the plan as tuples produced by a subquery are un-ordered.
- s.withNewPlan(removeTopLevelSort(newPlan))
- }
-
- // borrowed from OptimizeSubqueries in Spark
- private def removeTopLevelSort(plan: LogicalPlan): LogicalPlan = {
- if (!plan.containsPattern(SORT)) {
- return plan
- }
- plan match {
- case Sort(_, _, child) => child
- case Project(fields, child) => Project(fields, removeTopLevelSort(child))
- case other => other
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
deleted file mode 100644
index 8918dfec65..0000000000
--- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import java.util.Objects;
-
-public class Employee {
- private Integer id;
- private String dep;
-
- public Employee() {}
-
- public Employee(Integer id, String dep) {
- this.id = id;
- this.dep = dep;
- }
-
- public Integer getId() {
- return id;
- }
-
- public void setId(Integer id) {
- this.id = id;
- }
-
- public String getDep() {
- return dep;
- }
-
- public void setDep(String dep) {
- this.dep = dep;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- } else if (other == null || getClass() != other.getClass()) {
- return false;
- }
-
- Employee employee = (Employee) other;
- return Objects.equals(id, employee.id) && Objects.equals(dep, employee.dep);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id, dep);
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java
deleted file mode 100644
index de4acd74a7..0000000000
--- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.UUID;
-import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
-import org.apache.iceberg.PartitionStatisticsFile;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.PositionOutputStream;
-
-public class ProcedureUtil {
-
- private ProcedureUtil() {}
-
- static PartitionStatisticsFile writePartitionStatsFile(
- long snapshotId, String statsLocation, FileIO fileIO) {
- PositionOutputStream positionOutputStream;
- try {
- positionOutputStream = fileIO.newOutputFile(statsLocation).create();
- positionOutputStream.close();
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
-
- return ImmutableGenericPartitionStatisticsFile.builder()
- .snapshotId(snapshotId)
- .fileSizeInBytes(42L)
- .path(statsLocation)
- .build();
- }
-
- static String statsFileLocation(String tableLocation) {
- String statsFileName = "stats-file-" + UUID.randomUUID();
- return tableLocation.replaceFirst("file:", "") + "/metadata/" + statsFileName;
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java
deleted file mode 100644
index 4f137f5b8d..0000000000
--- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS;
-
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.iceberg.CatalogUtil;
-import org.apache.iceberg.hive.HiveCatalog;
-import org.apache.iceberg.hive.TestHiveMetastore;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.spark.SparkCatalogTestBase;
-import org.apache.iceberg.spark.SparkTestBase;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.internal.SQLConf;
-import org.junit.BeforeClass;
-
-public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase {
-
- private static final Random RANDOM = ThreadLocalRandom.current();
-
- public SparkExtensionsTestBase(
- String catalogName, String implementation, Map<String, String> config) {
- super(catalogName, implementation, config);
- }
-
- @BeforeClass
- public static void startMetastoreAndSpark() {
- SparkTestBase.metastore = new TestHiveMetastore();
- metastore.start();
- SparkTestBase.hiveConf = metastore.hiveConf();
-
- SparkTestBase.spark =
- SparkSession.builder()
- .master("local[2]")
- .config("spark.testing", "true")
- .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic")
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname))
- .config("spark.sql.shuffle.partitions", "4")
- .config("spark.sql.hive.metastorePartitionPruningFallbackOnException", "true")
- .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true")
- .config(
- SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean()))
- .enableHiveSupport()
- .getOrCreate();
-
- SparkTestBase.catalog =
- (HiveCatalog)
- CatalogUtil.loadCatalog(
- HiveCatalog.class.getName(), "hive", ImmutableMap.of(), hiveConf);
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
deleted file mode 100644
index 6aae084346..0000000000
--- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.apache.iceberg.DataOperations.DELETE;
-import static org.apache.iceberg.DataOperations.OVERWRITE;
-import static org.apache.iceberg.SnapshotSummary.ADDED_DELETE_FILES_PROP;
-import static org.apache.iceberg.SnapshotSummary.ADDED_FILES_PROP;
-import static org.apache.iceberg.SnapshotSummary.CHANGED_PARTITION_COUNT_PROP;
-import static org.apache.iceberg.SnapshotSummary.DELETED_FILES_PROP;
-import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
-import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE;
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.stream.Collectors;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.Files;
-import org.apache.iceberg.Snapshot;
-import org.apache.iceberg.SnapshotRef;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.data.GenericRecord;
-import org.apache.iceberg.data.parquet.GenericParquetWriter;
-import org.apache.iceberg.io.DataWriter;
-import org.apache.iceberg.io.OutputFile;
-import org.apache.iceberg.parquet.Parquet;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.spark.SparkCatalog;
-import org.apache.iceberg.spark.SparkSessionCatalog;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Encoder;
-import org.apache.spark.sql.Encoders;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.junit.Assert;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public abstract class SparkRowLevelOperationsTestBase extends SparkExtensionsTestBase {
-
- private static final Random RANDOM = ThreadLocalRandom.current();
-
- protected final String fileFormat;
- protected final boolean vectorized;
- protected final String distributionMode;
- protected final String branch;
-
- public SparkRowLevelOperationsTestBase(
- String catalogName,
- String implementation,
- Map<String, String> config,
- String fileFormat,
- boolean vectorized,
- String distributionMode,
- String branch) {
- super(catalogName, implementation, config);
- this.fileFormat = fileFormat;
- this.vectorized = vectorized;
- this.distributionMode = distributionMode;
- this.branch = branch;
- }
-
- @Parameters(
- name =
- "catalogName = {0}, implementation = {1}, config = {2},"
- + " format = {3}, vectorized = {4}, distributionMode = {5}, branch = {6}")
- public static Object[][] parameters() {
- return new Object[][] {
- {
- "testhive",
- SparkCatalog.class.getName(),
- ImmutableMap.of(
- "type", "hive",
- "default-namespace", "default"),
- "orc",
- true,
- WRITE_DISTRIBUTION_MODE_NONE,
- SnapshotRef.MAIN_BRANCH
- },
- {
- "testhive",
- SparkCatalog.class.getName(),
- ImmutableMap.of(
- "type", "hive",
- "default-namespace", "default"),
- "parquet",
- true,
- WRITE_DISTRIBUTION_MODE_NONE,
- "test",
- },
- {
- "testhadoop",
- SparkCatalog.class.getName(),
- ImmutableMap.of("type", "hadoop"),
- "parquet",
- RANDOM.nextBoolean(),
- WRITE_DISTRIBUTION_MODE_HASH,
- null
- },
- {
- "spark_catalog",
- SparkSessionCatalog.class.getName(),
- ImmutableMap.of(
- "type", "hive",
- "default-namespace", "default",
- "clients", "1",
- "parquet-enabled", "false",
- "cache-enabled",
- "false" // Spark will delete tables using v1, leaving the cache out of sync
- ),
- "avro",
- false,
- WRITE_DISTRIBUTION_MODE_RANGE,
- "test"
- }
- };
- }
-
- protected abstract Map<String, String> extraTableProperties();
-
- protected void initTable() {
- sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')", tableName, DEFAULT_FILE_FORMAT, fileFormat);
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')",
- tableName, WRITE_DISTRIBUTION_MODE, distributionMode);
-
- switch (fileFormat) {
- case "parquet":
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES('%s' '%b')",
- tableName, PARQUET_VECTORIZATION_ENABLED, vectorized);
- break;
- case "orc":
- Assert.assertTrue(vectorized);
- break;
- case "avro":
- Assert.assertFalse(vectorized);
- break;
- }
-
- Map<String, String> props = extraTableProperties();
- props.forEach(
- (prop, value) -> {
- sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')", tableName, prop, value);
- });
- }
-
- protected void createAndInitTable(String schema) {
- createAndInitTable(schema, null);
- }
-
- protected void createAndInitTable(String schema, String jsonData) {
- createAndInitTable(schema, "", jsonData);
- }
-
- protected void createAndInitTable(String schema, String partitioning, String jsonData) {
- sql("CREATE TABLE %s (%s) USING iceberg %s", tableName, schema, partitioning);
- initTable();
-
- if (jsonData != null) {
- try {
- Dataset<Row> ds = toDS(schema, jsonData);
- ds.coalesce(1).writeTo(tableName).append();
- createBranchIfNeeded();
- } catch (NoSuchTableException e) {
- throw new RuntimeException("Failed to write data", e);
- }
- }
- }
-
- protected void append(String table, String jsonData) {
- append(table, null, jsonData);
- }
-
- protected void append(String table, String schema, String jsonData) {
- try {
- Dataset<Row> ds = toDS(schema, jsonData);
- ds.coalesce(1).writeTo(table).append();
- } catch (NoSuchTableException e) {
- throw new RuntimeException("Failed to write data", e);
- }
- }
-
- protected void createOrReplaceView(String name, String jsonData) {
- createOrReplaceView(name, null, jsonData);
- }
-
- protected void createOrReplaceView(String name, String schema, String jsonData) {
- Dataset<Row> ds = toDS(schema, jsonData);
- ds.createOrReplaceTempView(name);
- }
-
- protected <T> void createOrReplaceView(String name, List<T> data, Encoder<T> encoder) {
- spark.createDataset(data, encoder).createOrReplaceTempView(name);
- }
-
- private Dataset<Row> toDS(String schema, String jsonData) {
- List<String> jsonRows =
- Arrays.stream(jsonData.split("\n"))
- .filter(str -> !str.trim().isEmpty())
- .collect(Collectors.toList());
- Dataset<String> jsonDS = spark.createDataset(jsonRows, Encoders.STRING());
-
- if (schema != null) {
- return spark.read().schema(schema).json(jsonDS);
- } else {
- return spark.read().json(jsonDS);
- }
- }
-
- protected void validateDelete(
- Snapshot snapshot, String changedPartitionCount, String deletedDataFiles) {
- validateSnapshot(snapshot, DELETE, changedPartitionCount, deletedDataFiles, null, null);
- }
-
- protected void validateCopyOnWrite(
- Snapshot snapshot,
- String changedPartitionCount,
- String deletedDataFiles,
- String addedDataFiles) {
- String operation = null == addedDataFiles && null != deletedDataFiles ? DELETE : OVERWRITE;
- validateSnapshot(
- snapshot, operation, changedPartitionCount, deletedDataFiles, null, addedDataFiles);
- }
-
- protected void validateMergeOnRead(
- Snapshot snapshot,
- String changedPartitionCount,
- String addedDeleteFiles,
- String addedDataFiles) {
- String operation = null == addedDataFiles && null != addedDeleteFiles ? DELETE : OVERWRITE;
- validateSnapshot(
- snapshot, operation, changedPartitionCount, null, addedDeleteFiles, addedDataFiles);
- }
-
- protected void validateSnapshot(
- Snapshot snapshot,
- String operation,
- String changedPartitionCount,
- String deletedDataFiles,
- String addedDeleteFiles,
- String addedDataFiles) {
- Assert.assertEquals("Operation must match", operation, snapshot.operation());
- validateProperty(snapshot, CHANGED_PARTITION_COUNT_PROP, changedPartitionCount);
- validateProperty(snapshot, DELETED_FILES_PROP, deletedDataFiles);
- validateProperty(snapshot, ADDED_DELETE_FILES_PROP, addedDeleteFiles);
- validateProperty(snapshot, ADDED_FILES_PROP, addedDataFiles);
- }
-
- protected void validateProperty(Snapshot snapshot, String property, Set<String> expectedValues) {
- String actual = snapshot.summary().get(property);
- Assert.assertTrue(
- "Snapshot property "
- + property
- + " has unexpected value, actual = "
- + actual
- + ", expected one of : "
- + String.join(",", expectedValues),
- expectedValues.contains(actual));
- }
-
- protected void validateProperty(Snapshot snapshot, String property, String expectedValue) {
- if (null == expectedValue) {
- assertThat(snapshot.summary()).doesNotContainKey(property);
- } else {
- assertThat(snapshot.summary())
- .as("Snapshot property " + property + " has unexpected value.")
- .containsEntry(property, expectedValue);
- }
- }
-
- protected void sleep(long millis) {
- try {
- Thread.sleep(millis);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
- protected DataFile writeDataFile(Table table, List<GenericRecord> records) {
- try {
- OutputFile file = Files.localOutput(temp.newFile());
-
- DataWriter<GenericRecord> dataWriter =
- Parquet.writeData(file)
- .forTable(table)
- .createWriterFunc(GenericParquetWriter::buildWriter)
- .overwrite()
- .build();
-
- try {
- for (GenericRecord record : records) {
- dataWriter.write(record);
- }
- } finally {
- dataWriter.close();
- }
-
- return dataWriter.toDataFile();
-
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- @Override
- protected String commitTarget() {
- return branch == null ? tableName : String.format("%s.branch_%s", tableName, branch);
- }
-
- @Override
- protected String selectTarget() {
- return branch == null ? tableName : String.format("%s VERSION AS OF '%s'", tableName, branch);
- }
-
- protected void createBranchIfNeeded() {
- if (branch != null && !branch.equals(SnapshotRef.MAIN_BRANCH)) {
- sql("ALTER TABLE %s CREATE BRANCH %s", tableName, branch);
- }
- }
-}
diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
deleted file mode 100644
index b24ebfcf79..0000000000
--- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
+++ /dev/null
@@ -1,1161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaBuilder;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumWriter;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.spark.SparkCatalogConfig;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.types.DataTypes;
-import org.apache.spark.sql.types.Metadata;
-import org.apache.spark.sql.types.StructField;
-import org.apache.spark.sql.types.StructType;
-import org.joda.time.DateTime;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runners.Parameterized.Parameters;
-
-public class TestAddFilesProcedure extends SparkExtensionsTestBase {
-
- @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}, formatVersion = {3}")
- public static Object[][] parameters() {
- return new Object[][] {
- {
- SparkCatalogConfig.HIVE.catalogName(),
- SparkCatalogConfig.HIVE.implementation(),
- SparkCatalogConfig.HIVE.properties(),
- 1
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
- 2
- },
- {
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
- 2
- }
- };
- }
-
- private final int formatVersion;
- private final String sourceTableName = "source_table";
- private File fileTableDir;
-
- public TestAddFilesProcedure(
- String catalogName, String implementation, Map<String, String> config, int formatVersion) {
- super(catalogName, implementation, config);
- this.formatVersion = formatVersion;
- }
-
- @Rule public TemporaryFolder temp = new TemporaryFolder();
-
- @Before
- public void setupTempDirs() {
- try {
- fileTableDir = temp.newFolder();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @After
- public void dropTables() {
- sql("DROP TABLE IF EXISTS %s", sourceTableName);
- sql("DROP TABLE IF EXISTS %s", tableName);
- }
-
- @Test
- public void addDataUnpartitioned() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void deleteAndAddBackUnpartitioned() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- String deleteData = "DELETE FROM %s";
- sql(deleteData, tableName);
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Ignore // TODO Classpath issues prevent us from actually writing to a Spark ORC table
- public void addDataUnpartitionedOrc() {
- createUnpartitionedFileTable("orc");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`orc`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- Assert.assertEquals(2L, result);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addAvroFile() throws Exception {
- // Spark Session Catalog cannot load metadata tables
- // with "The namespace in session catalog must have exactly one name part"
- Assume.assumeFalse(catalogName.equals("spark_catalog"));
-
- // Create an Avro file
-
- Schema schema =
- SchemaBuilder.record("record")
- .fields()
- .requiredInt("id")
- .requiredString("data")
- .endRecord();
- GenericRecord record1 = new GenericData.Record(schema);
- record1.put("id", 1L);
- record1.put("data", "a");
- GenericRecord record2 = new GenericData.Record(schema);
- record2.put("id", 2L);
- record2.put("data", "b");
- File outputFile = temp.newFile("test.avro");
-
- DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter(schema);
- DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter(datumWriter);
- dataFileWriter.create(schema, outputFile);
- dataFileWriter.append(record1);
- dataFileWriter.append(record2);
- dataFileWriter.close();
-
- createIcebergTable("id Long, data String");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`avro`.`%s`')",
- catalogName, tableName, outputFile.getPath());
- assertOutput(result, 1L, 1L);
-
- List<Object[]> expected = Lists.newArrayList(new Object[] {1L, "a"}, new Object[] {2L, "b"});
-
- assertEquals(
- "Iceberg table contains correct data",
- expected,
- sql("SELECT * FROM %s ORDER BY id", tableName));
-
- List<Object[]> actualRecordCount =
- sql("select %s from %s.files", DataFile.RECORD_COUNT.name(), tableName);
- List<Object[]> expectedRecordCount = Lists.newArrayList();
- expectedRecordCount.add(new Object[] {2L});
- assertEquals(
- "Iceberg file metadata should have correct metadata count",
- expectedRecordCount,
- actualRecordCount);
- }
-
- // TODO Adding spark-avro doesn't work in tests
- @Ignore
- public void addDataUnpartitionedAvro() {
- createUnpartitionedFileTable("avro");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`avro`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- Assert.assertEquals(2L, result);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addDataUnpartitionedHive() {
- createUnpartitionedHiveTable();
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- List<Object[]> result =
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addDataUnpartitionedExtraCol() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String, foo string");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addDataUnpartitionedMissingCol() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addDataPartitionedMissingCol() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 8L, 4L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addDataPartitioned() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 8L, 4L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Ignore // TODO Classpath issues prevent us from actually writing to a Spark ORC table
- public void addDataPartitionedOrc() {
- createPartitionedFileTable("orc");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- Assert.assertEquals(8L, result);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- // TODO Adding spark-avro doesn't work in tests
- @Ignore
- public void addDataPartitionedAvro() {
- createPartitionedFileTable("avro");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`avro`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- Assert.assertEquals(8L, result);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addDataPartitionedHive() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
-
- assertOutput(result, 8L, 4L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addPartitionToPartitioned() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void deleteAndAddBackPartitioned() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- String deleteData = "DELETE FROM %s where id = 1";
- sql(deleteData, tableName);
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addPartitionToPartitionedSnapshotIdInheritanceEnabledInTwoRuns() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')",
- tableName, TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED);
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 2))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id < 3 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
-
- // verify manifest file name has uuid pattern
- String manifestPath = (String) sql("select path from %s.manifests", tableName).get(0)[0];
-
- Pattern uuidPattern = Pattern.compile("[a-f0-9]{8}(?:-[a-f0-9]{4}){4}[a-f0-9]{8}");
-
- Matcher matcher = uuidPattern.matcher(manifestPath);
- Assert.assertTrue("verify manifest path has uuid", matcher.find());
- }
-
- @Test
- public void addDataPartitionedByDateToPartitioned() {
- createDatePartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, date Date", "PARTITIONED BY (date)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('date', '2021-01-01'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, date FROM %s WHERE date = '2021-01-01' ORDER BY id", sourceTableName),
- sql("SELECT id, name, date FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addDataPartitionedVerifyPartitionTypeInferredCorrectly() {
- createTableWithTwoPartitions("parquet");
-
- createIcebergTable(
- "id Integer, name String, date Date, dept String", "PARTITIONED BY (date, dept)");
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('date', '2021-01-01'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- String sqlFormat =
- "SELECT id, name, dept, date FROM %s WHERE date = '2021-01-01' and dept= '01' ORDER BY id";
- assertEquals(
- "Iceberg table contains correct data",
- sql(sqlFormat, sourceTableName),
- sql(sqlFormat, tableName));
- }
-
- @Test
- public void addFilteredPartitionsToPartitioned() {
- createCompositePartitionedTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addFilteredPartitionsToPartitioned2() {
- createCompositePartitionedTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 6L, 3L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql(
- "SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id",
- sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnId() {
- createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnDept() {
- createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 6L, 3L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql(
- "SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id",
- sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addWeirdCaseHiveTable() {
- createWeirdCaseTable();
-
- createIcebergTable(
- "id Integer, `naMe` String, dept String, subdept String", "PARTITIONED BY (`naMe`)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '%s', map('naMe', 'John Doe'))",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result, 2L, 1L);
-
- /*
- While we would like to use
- SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id
- Spark does not actually handle this pushdown correctly for hive based tables and it returns 0 records
- */
- List<Object[]> expected =
- sql("SELECT id, `naMe`, dept, subdept from %s ORDER BY id", sourceTableName).stream()
- .filter(r -> r[1].equals("John Doe"))
- .collect(Collectors.toList());
-
- // TODO when this assert breaks Spark fixed the pushdown issue
- Assert.assertEquals(
- "If this assert breaks it means that Spark has fixed the pushdown issue",
- 0,
- sql(
- "SELECT id, `naMe`, dept, subdept from %s WHERE `naMe` = 'John Doe' ORDER BY id",
- sourceTableName)
- .size());
-
- // Pushdown works for iceberg
- Assert.assertEquals(
- "We should be able to pushdown mixed case partition keys",
- 2,
- sql(
- "SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id",
- tableName)
- .size());
-
- assertEquals(
- "Iceberg table contains correct data",
- expected,
- sql("SELECT id, `naMe`, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void addPartitionToPartitionedHive() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '%s', map('id', 1))",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Test
- public void invalidDataImport() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .as("Should forbid adding of partitioned data to unpartitioned table")
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Cannot use partition filter with an unpartitioned table");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .as("Should forbid adding of partitioned data to unpartitioned table")
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Cannot add partitioned files to an unpartitioned table");
- }
-
- @Test
- public void invalidDataImportPartitioned() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('x', '1', 'y', '2'))",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .as("Should forbid adding with a mismatching partition spec")
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("is greater than the number of partitioned columns");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', '2'))",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .as("Should forbid adding with partition spec with incorrect columns")
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining(
- "specified partition filter refers to columns that are not partitioned");
- }
-
- @Test
- public void addTwice() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result1 =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1))",
- catalogName, tableName, sourceTableName);
- assertOutput(result1, 2L, 1L);
-
- List<Object[]> result2 =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 2))",
- catalogName, tableName, sourceTableName);
- assertOutput(result2, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", tableName));
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", tableName));
- }
-
- @Test
- public void duplicateDataPartitioned() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1))",
- catalogName, tableName, sourceTableName);
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1))",
- catalogName, tableName, sourceTableName))
- .as("Should not allow adding duplicate files")
- .isInstanceOf(IllegalStateException.class)
- .hasMessageContaining(
- "Cannot complete import because data files to be imported already"
- + " exist within the target table");
- }
-
- @Test
- public void duplicateDataPartitionedAllowed() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
... 104128 lines suppressed ...