You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2025/12/23 03:15:01 UTC
(iceberg) 02/03: Spark: Copy back 4.1 as 4.0
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit ddea5658ece8ba5aff6f22d8e3477a05064175d1
Author: manuzhang <ow...@gmail.com>
AuthorDate: Tue Dec 23 09:36:43 2025 +0800
Spark: Copy back 4.1 as 4.0
---
spark/v4.0/build.gradle | 349 +++
.../apache/iceberg/DeleteFileIndexBenchmark.java | 279 ++
.../spark/MergeCardinalityCheckBenchmark.java | 231 ++
.../apache/iceberg/spark/PlanningBenchmark.java | 409 +++
.../iceberg/spark/TaskGroupPlanningBenchmark.java | 273 ++
.../iceberg/spark/UpdateProjectionBenchmark.java | 213 ++
.../IcebergSqlExtensions.g4 | 367 +++
.../extensions/IcebergSparkSessionExtensions.scala | 44 +
.../spark/sql/catalyst/analysis/CheckViews.scala | 135 +
.../spark/sql/catalyst/analysis/ResolveViews.scala | 185 ++
.../catalyst/analysis/RewriteViewCommands.scala | 210 ++
.../spark/sql/catalyst/analysis/ViewUtil.scala | 50 +
.../catalyst/optimizer/ReplaceStaticInvoke.scala | 120 +
.../IcebergSparkSqlExtensionsParser.scala | 342 +++
.../IcebergSqlExtensionsAstBuilder.scala | 385 +++
.../catalyst/plans/logical/AddPartitionField.scala | 33 +
.../sql/catalyst/plans/logical/BranchOptions.scala | 25 +
.../plans/logical/CreateOrReplaceBranch.scala | 39 +
.../plans/logical/CreateOrReplaceTag.scala | 39 +
.../sql/catalyst/plans/logical/DropBranch.scala | 32 +
.../plans/logical/DropIdentifierFields.scala | 31 +
.../plans/logical/DropPartitionField.scala | 32 +
.../spark/sql/catalyst/plans/logical/DropTag.scala | 32 +
.../plans/logical/ReplacePartitionField.scala | 38 +
.../plans/logical/SetIdentifierFields.scala | 31 +
.../sql/catalyst/plans/logical/TagOptions.scala | 21 +
.../plans/logical/views/CreateIcebergView.scala | 53 +
.../plans/logical/views/DropIcebergView.scala | 27 +
.../plans/logical/views/ResolvedV2View.scala | 29 +
.../plans/logical/views/ShowIcebergViews.scala | 35 +
.../datasources/v2/AddPartitionFieldExec.scala | 58 +
.../v2/AlterV2ViewSetPropertiesExec.scala | 48 +
.../v2/AlterV2ViewUnsetPropertiesExec.scala | 53 +
.../datasources/v2/CreateOrReplaceBranchExec.scala | 103 +
.../datasources/v2/CreateOrReplaceTagExec.scala | 86 +
.../datasources/v2/CreateV2ViewExec.scala | 128 +
.../datasources/v2/DescribeV2ViewExec.scala | 79 +
.../execution/datasources/v2/DropBranchExec.scala | 56 +
.../datasources/v2/DropIdentifierFieldsExec.scala | 68 +
.../datasources/v2/DropPartitionFieldExec.scala | 68 +
.../sql/execution/datasources/v2/DropTagExec.scala | 52 +
.../execution/datasources/v2/DropV2ViewExec.scala | 44 +
.../v2/ExtendedDataSourceV2Strategy.scala | 192 ++
.../datasources/v2/RenameV2ViewExec.scala | 40 +
.../datasources/v2/ReplacePartitionFieldExec.scala | 76 +
.../datasources/v2/SetIdentifierFieldsExec.scala | 52 +
.../v2/SetWriteDistributionAndOrderingExec.scala | 85 +
.../datasources/v2/ShowCreateV2ViewExec.scala | 78 +
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 51 +
.../execution/datasources/v2/ShowV2ViewsExec.scala | 70 +
.../apache/iceberg/spark/TestExtendedParser.java | 231 ++
.../apache/iceberg/spark/extensions/Employee.java | 66 +
.../spark/extensions/ExtensionsTestBase.java | 72 +
.../iceberg/spark/extensions/ProcedureUtil.java | 54 +
.../iceberg/spark/extensions/SparkPlanUtil.java | 107 +
.../SparkRowLevelOperationsTestBase.java | 461 +++
.../spark/extensions/TestAddFilesProcedure.java | 1479 ++++++++++
.../extensions/TestAlterTablePartitionFields.java | 650 +++++
.../spark/extensions/TestAlterTableSchema.java | 148 +
.../spark/extensions/TestAncestorsOfProcedure.java | 163 ++
.../iceberg/spark/extensions/TestBranchDDL.java | 432 +++
.../spark/extensions/TestChangelogTable.java | 453 +++
.../TestCherrypickSnapshotProcedure.java | 194 ++
.../TestComputePartitionStatsProcedure.java | 121 +
.../extensions/TestComputeTableStatsProcedure.java | 137 +
.../spark/extensions/TestConflictValidation.java | 394 +++
.../spark/extensions/TestCopyOnWriteDelete.java | 227 ++
.../spark/extensions/TestCopyOnWriteMerge.java | 195 ++
.../spark/extensions/TestCopyOnWriteUpdate.java | 183 ++
.../extensions/TestCopyOnWriteWithLineage.java | 35 +
.../TestCreateChangelogViewProcedure.java | 694 +++++
.../iceberg/spark/extensions/TestDelete.java | 1486 ++++++++++
.../extensions/TestExpireSnapshotsProcedure.java | 624 ++++
.../extensions/TestFastForwardBranchProcedure.java | 264 ++
.../apache/iceberg/spark/extensions/TestMerge.java | 3046 ++++++++++++++++++++
.../spark/extensions/TestMergeOnReadDelete.java | 352 +++
.../spark/extensions/TestMergeOnReadMerge.java | 173 ++
.../spark/extensions/TestMergeOnReadUpdate.java | 226 ++
.../extensions/TestMergeOnReadWithLineage.java | 35 +
.../TestMetaColumnProjectionWithStageScan.java | 125 +
.../spark/extensions/TestMetadataTables.java | 993 +++++++
.../extensions/TestMigrateTableProcedure.java | 291 ++
.../extensions/TestPublishChangesProcedure.java | 187 ++
.../extensions/TestRegisterTableProcedure.java | 85 +
.../spark/extensions/TestRemoteScanPlanning.java | 58 +
.../extensions/TestRemoveOrphanFilesProcedure.java | 752 +++++
.../spark/extensions/TestReplaceBranch.java | 270 ++
.../TestRequiredDistributionAndOrdering.java | 313 ++
.../extensions/TestRewriteDataFilesProcedure.java | 1108 +++++++
.../extensions/TestRewriteManifestsProcedure.java | 448 +++
.../extensions/TestRewritePositionDeleteFiles.java | 418 +++
.../TestRewritePositionDeleteFilesProcedure.java | 250 ++
.../extensions/TestRewriteTablePathProcedure.java | 270 ++
.../TestRollbackToSnapshotProcedure.java | 286 ++
.../TestRollbackToTimestampProcedure.java | 339 +++
.../TestRowLevelOperationsWithLineage.java | 631 ++++
.../TestSetCurrentSnapshotProcedure.java | 291 ++
.../TestSetWriteDistributionAndOrdering.java | 339 +++
.../extensions/TestSnapshotTableProcedure.java | 378 +++
...toragePartitionedJoinsInRowLevelOperations.java | 316 ++
.../extensions/TestSystemFunctionPushDownDQL.java | 313 ++
...SystemFunctionPushDownInRowLevelOperations.java | 365 +++
.../iceberg/spark/extensions/TestTagDDL.java | 365 +++
.../iceberg/spark/extensions/TestUpdate.java | 1518 ++++++++++
.../apache/iceberg/spark/extensions/TestViews.java | 2121 ++++++++++++++
.../iceberg/spark/extensions/TestWriteAborts.java | 185 ++
.../org/apache/iceberg/spark/source/PlanUtils.java | 99 +
spark/v4.0/spark-runtime/LICENSE | 530 ++++
spark/v4.0/spark-runtime/NOTICE | 403 +++
.../org/apache/iceberg/spark/TestRoundTrip.java | 182 ++
.../apache/iceberg/spark/SparkBenchmarkUtil.java | 58 +
.../spark/action/DeleteOrphanFilesBenchmark.java | 189 ++
.../action/IcebergSortCompactionBenchmark.java | 404 +++
.../iceberg/spark/action/RandomGeneratingUDF.java | 50 +
.../SparkParquetReadersFlatDataBenchmark.java | 229 ++
.../SparkParquetReadersNestedDataBenchmark.java | 227 ++
.../SparkParquetWritersFlatDataBenchmark.java | 130 +
.../SparkParquetWritersNestedDataBenchmark.java | 130 +
.../org/apache/iceberg/spark/source/Action.java | 24 +
.../iceberg/spark/source/DVReaderBenchmark.java | 267 ++
.../iceberg/spark/source/DVWriterBenchmark.java | 251 ++
.../spark/source/IcebergSourceBenchmark.java | 209 ++
.../spark/source/IcebergSourceDeleteBenchmark.java | 340 +++
.../source/IcebergSourceFlatDataBenchmark.java | 59 +
.../source/IcebergSourceNestedDataBenchmark.java | 59 +
.../IcebergSourceNestedListDataBenchmark.java | 62 +
.../iceberg/spark/source/WritersBenchmark.java | 393 +++
.../spark/source/avro/AvroWritersBenchmark.java | 39 +
.../IcebergSourceFlatAvroDataReadBenchmark.java | 142 +
.../IcebergSourceNestedAvroDataReadBenchmark.java | 142 +
.../orc/IcebergSourceFlatORCDataBenchmark.java | 68 +
.../orc/IcebergSourceFlatORCDataReadBenchmark.java | 210 ++
...ebergSourceNestedListORCDataWriteBenchmark.java | 109 +
.../IcebergSourceNestedORCDataReadBenchmark.java | 183 ++
...cebergSourceFlatParquetDataFilterBenchmark.java | 129 +
.../IcebergSourceFlatParquetDataReadBenchmark.java | 165 ++
...IcebergSourceFlatParquetDataWriteBenchmark.java | 89 +
...gSourceNestedListParquetDataWriteBenchmark.java | 89 +
...bergSourceNestedParquetDataFilterBenchmark.java | 128 +
...cebergSourceNestedParquetDataReadBenchmark.java | 166 ++
...ebergSourceNestedParquetDataWriteBenchmark.java | 88 +
.../IcebergSourceParquetEqDeleteBenchmark.java | 57 +
...ebergSourceParquetMultiDeleteFileBenchmark.java | 57 +
.../IcebergSourceParquetPosDeleteBenchmark.java | 60 +
...gSourceParquetWithUnrelatedDeleteBenchmark.java | 64 +
.../source/parquet/ParquetWritersBenchmark.java | 39 +
...dDictionaryEncodedFlatParquetDataBenchmark.java | 127 +
.../VectorizedReadFlatParquetDataBenchmark.java | 355 +++
.../VectorizedReadParquetDecimalBenchmark.java | 207 ++
.../apache/iceberg/SparkDistributedDataScan.java | 272 ++
.../java/org/apache/iceberg/spark/BaseCatalog.java | 94 +
.../iceberg/spark/BaseFileRewriteCoordinator.java | 81 +
.../apache/iceberg/spark/ChangelogIterator.java | 127 +
.../org/apache/iceberg/spark/CommitMetadata.java | 66 +
.../iceberg/spark/ComputeUpdateIterator.java | 140 +
.../org/apache/iceberg/spark/ExtendedParser.java | 110 +
.../iceberg/spark/FileRewriteCoordinator.java | 32 +
.../org/apache/iceberg/spark/IcebergSpark.java | 56 +
.../org/apache/iceberg/spark/JobGroupInfo.java | 48 +
.../org/apache/iceberg/spark/JobGroupUtils.java | 64 +
.../org/apache/iceberg/spark/OrcBatchReadConf.java | 27 +
.../apache/iceberg/spark/ParquetBatchReadConf.java | 29 +
.../apache/iceberg/spark/ParquetReaderType.java | 47 +
.../org/apache/iceberg/spark/PathIdentifier.java | 57 +
.../spark/PositionDeletesRewriteCoordinator.java | 33 +
.../spark/PruneColumnsWithoutReordering.java | 249 ++
.../iceberg/spark/RemoveCarryoverIterator.java | 148 +
.../iceberg/spark/RemoveNetCarryoverIterator.java | 129 +
.../apache/iceberg/spark/RollbackStagedTable.java | 142 +
.../apache/iceberg/spark/ScanTaskSetManager.java | 73 +
.../org/apache/iceberg/spark/SortOrderToSpark.java | 101 +
.../java/org/apache/iceberg/spark/Spark3Util.java | 1079 +++++++
.../org/apache/iceberg/spark/SparkAggregates.java | 84 +
.../iceberg/spark/SparkCachedTableCatalog.java | 257 ++
.../org/apache/iceberg/spark/SparkCatalog.java | 1049 +++++++
.../apache/iceberg/spark/SparkCompressionUtil.java | 148 +
.../org/apache/iceberg/spark/SparkConfParser.java | 320 ++
.../org/apache/iceberg/spark/SparkContentFile.java | 277 ++
.../org/apache/iceberg/spark/SparkDataFile.java | 46 +
.../org/apache/iceberg/spark/SparkDeleteFile.java | 40 +
.../apache/iceberg/spark/SparkExceptionUtil.java | 64 +
.../apache/iceberg/spark/SparkExecutorCache.java | 228 ++
.../org/apache/iceberg/spark/SparkFilters.java | 280 ++
.../org/apache/iceberg/spark/SparkFixupTypes.java | 63 +
.../apache/iceberg/spark/SparkFunctionCatalog.java | 45 +
.../org/apache/iceberg/spark/SparkReadConf.java | 378 +++
.../org/apache/iceberg/spark/SparkReadOptions.java | 98 +
.../apache/iceberg/spark/SparkSQLProperties.java | 112 +
.../org/apache/iceberg/spark/SparkSchemaUtil.java | 372 +++
.../apache/iceberg/spark/SparkSessionCatalog.java | 542 ++++
.../org/apache/iceberg/spark/SparkStructLike.java | 54 +
.../org/apache/iceberg/spark/SparkTableCache.java | 54 +
.../org/apache/iceberg/spark/SparkTableUtil.java | 1311 +++++++++
.../org/apache/iceberg/spark/SparkTypeToType.java | 171 ++
.../org/apache/iceberg/spark/SparkTypeVisitor.java | 86 +
.../java/org/apache/iceberg/spark/SparkUtil.java | 339 +++
.../org/apache/iceberg/spark/SparkV2Filters.java | 484 ++++
.../apache/iceberg/spark/SparkValueConverter.java | 157 +
.../org/apache/iceberg/spark/SparkWriteConf.java | 732 +++++
.../apache/iceberg/spark/SparkWriteOptions.java | 88 +
.../iceberg/spark/SparkWriteRequirements.java | 59 +
.../org/apache/iceberg/spark/SparkWriteUtil.java | 259 ++
.../apache/iceberg/spark/SupportsFunctions.java | 65 +
.../apache/iceberg/spark/SupportsReplaceView.java | 56 +
.../org/apache/iceberg/spark/TypeToSparkType.java | 166 ++
.../actions/BaseSnapshotUpdateSparkAction.java | 47 +
.../iceberg/spark/actions/BaseSparkAction.java | 443 +++
.../actions/BaseTableCreationSparkAction.java | 182 ++
.../actions/ComputePartitionStatsSparkAction.java | 103 +
.../actions/ComputeTableStatsSparkAction.java | 179 ++
.../actions/DeleteOrphanFilesSparkAction.java | 620 ++++
.../actions/DeleteReachableFilesSparkAction.java | 156 +
.../spark/actions/ExpireSnapshotsSparkAction.java | 286 ++
.../org/apache/iceberg/spark/actions/FileInfo.java | 52 +
.../iceberg/spark/actions/ManifestFileBean.java | 191 ++
.../spark/actions/MigrateTableSparkAction.java | 259 ++
.../iceberg/spark/actions/NDVSketchUtil.java | 91 +
.../actions/RemoveDanglingDeletesSparkAction.java | 206 ++
.../spark/actions/RewriteDataFilesSparkAction.java | 448 +++
.../spark/actions/RewriteManifestsSparkAction.java | 606 ++++
.../RewritePositionDeleteFilesSparkAction.java | 380 +++
.../spark/actions/RewriteTablePathSparkAction.java | 821 ++++++
.../iceberg/spark/actions/SetAccumulator.java | 61 +
.../spark/actions/SnapshotTableSparkAction.java | 225 ++
.../apache/iceberg/spark/actions/SparkActions.java | 122 +
.../actions/SparkBinPackFileRewriteRunner.java | 71 +
.../spark/actions/SparkDataFileRewriteRunner.java | 61 +
.../actions/SparkRewritePositionDeleteRunner.java | 150 +
.../iceberg/spark/actions/SparkRewriteRunner.java | 82 +
.../actions/SparkShufflingDataRewritePlanner.java | 82 +
.../actions/SparkShufflingFileRewriteRunner.java | 228 ++
.../spark/actions/SparkSortFileRewriteRunner.java | 64 +
.../actions/SparkZOrderFileRewriteRunner.java | 204 ++
.../iceberg/spark/actions/SparkZOrderUDF.java | 325 +++
.../spark/data/AvroWithSparkSchemaVisitor.java | 81 +
.../spark/data/ParquetWithSparkSchemaVisitor.java | 248 ++
.../apache/iceberg/spark/data/SparkAvroWriter.java | 170 ++
.../apache/iceberg/spark/data/SparkOrcReader.java | 135 +
.../iceberg/spark/data/SparkOrcValueReaders.java | 261 ++
.../iceberg/spark/data/SparkOrcValueWriters.java | 221 ++
.../apache/iceberg/spark/data/SparkOrcWriter.java | 247 ++
.../iceberg/spark/data/SparkParquetReaders.java | 782 +++++
.../iceberg/spark/data/SparkParquetWriters.java | 650 +++++
.../iceberg/spark/data/SparkPlannedAvroReader.java | 198 ++
.../iceberg/spark/data/SparkValueReaders.java | 352 +++
.../iceberg/spark/data/SparkValueWriters.java | 280 ++
.../vectorized/ArrowVectorAccessorFactory.java | 140 +
.../data/vectorized/ArrowVectorAccessors.java | 38 +
.../spark/data/vectorized/ColumnVectorBuilder.java | 45 +
.../data/vectorized/ColumnVectorWithFilter.java | 155 +
.../spark/data/vectorized/ColumnarBatchReader.java | 94 +
.../spark/data/vectorized/ColumnarBatchUtil.java | 181 ++
.../spark/data/vectorized/CometColumnReader.java | 140 +
.../data/vectorized/CometColumnarBatchReader.java | 137 +
.../data/vectorized/CometConstantColumnReader.java | 65 +
.../data/vectorized/CometDeleteColumnReader.java | 74 +
.../data/vectorized/CometDeletedColumnVector.java | 155 +
.../data/vectorized/CometPositionColumnReader.java | 62 +
.../vectorized/CometVectorizedReaderBuilder.java | 138 +
.../data/vectorized/ConstantColumnVector.java | 141 +
.../spark/data/vectorized/DeletedColumnVector.java | 123 +
.../data/vectorized/IcebergArrowColumnVector.java | 164 ++
.../data/vectorized/RowPositionColumnVector.java | 120 +
.../vectorized/UpdatableDeletedColumnVector.java | 23 +
.../data/vectorized/VectorizedSparkOrcReaders.java | 492 ++++
.../vectorized/VectorizedSparkParquetReaders.java | 143 +
.../spark/functions/BaseScalarFunction.java | 40 +
.../iceberg/spark/functions/BucketFunction.java | 366 +++
.../iceberg/spark/functions/DaysFunction.java | 152 +
.../iceberg/spark/functions/HoursFunction.java | 146 +
.../spark/functions/IcebergVersionFunction.java | 95 +
.../iceberg/spark/functions/MonthsFunction.java | 143 +
.../iceberg/spark/functions/SparkFunctions.java | 75 +
.../iceberg/spark/functions/TruncateFunction.java | 357 +++
.../spark/functions/UnaryUnboundFunction.java | 44 +
.../iceberg/spark/functions/YearsFunction.java | 143 +
.../spark/procedures/AddFilesProcedure.java | 278 ++
.../spark/procedures/AncestorsOfProcedure.java | 125 +
.../iceberg/spark/procedures/BaseProcedure.java | 289 ++
.../procedures/CherrypickSnapshotProcedure.java | 116 +
.../procedures/ComputePartitionStatsProcedure.java | 127 +
.../procedures/ComputeTableStatsProcedure.java | 131 +
.../procedures/CreateChangelogViewProcedure.java | 329 +++
.../spark/procedures/ExpireSnapshotsProcedure.java | 204 ++
.../procedures/FastForwardBranchProcedure.java | 109 +
.../spark/procedures/MigrateTableProcedure.java | 133 +
.../iceberg/spark/procedures/ProcedureInput.java | 274 ++
.../spark/procedures/PublishChangesProcedure.java | 129 +
.../spark/procedures/RegisterTableProcedure.java | 126 +
.../procedures/RemoveOrphanFilesProcedure.java | 252 ++
.../procedures/RewriteDataFilesProcedure.java | 229 ++
.../procedures/RewriteManifestsProcedure.java | 135 +
.../RewritePositionDeleteFilesProcedure.java | 131 +
.../procedures/RewriteTablePathProcedure.java | 157 +
.../procedures/RollbackToSnapshotProcedure.java | 112 +
.../procedures/RollbackToTimestampProcedure.java | 116 +
.../procedures/SetCurrentSnapshotProcedure.java | 130 +
.../spark/procedures/SnapshotTableProcedure.java | 132 +
.../iceberg/spark/procedures/SparkProcedures.java | 71 +
.../iceberg/spark/source/BaseBatchReader.java | 221 ++
.../apache/iceberg/spark/source/BaseReader.java | 256 ++
.../apache/iceberg/spark/source/BaseRowReader.java | 127 +
.../iceberg/spark/source/BatchDataReader.java | 126 +
.../iceberg/spark/source/ChangelogRowReader.java | 176 ++
.../apache/iceberg/spark/source/DVIterator.java | 109 +
.../spark/source/EqualityDeleteRowReader.java | 57 +
.../iceberg/spark/source/ExtractRowLineage.java | 92 +
.../iceberg/spark/source/HasIcebergCatalog.java | 30 +
.../apache/iceberg/spark/source/IcebergSource.java | 265 ++
.../iceberg/spark/source/InternalRowWrapper.java | 109 +
.../spark/source/PositionDeletesRowReader.java | 120 +
.../apache/iceberg/spark/source/RowDataReader.java | 125 +
.../spark/source/SerializableTableWithSize.java | 108 +
.../apache/iceberg/spark/source/SparkBatch.java | 232 ++
.../iceberg/spark/source/SparkBatchQueryScan.java | 293 ++
.../iceberg/spark/source/SparkChangelogScan.java | 174 ++
.../iceberg/spark/source/SparkChangelogTable.java | 140 +
.../iceberg/spark/source/SparkCleanupUtil.java | 141 +
.../spark/source/SparkColumnStatistics.java | 88 +
.../spark/source/SparkColumnarReaderFactory.java | 71 +
.../spark/source/SparkCopyOnWriteOperation.java | 115 +
.../iceberg/spark/source/SparkCopyOnWriteScan.java | 191 ++
.../spark/source/SparkFileWriterFactory.java | 343 +++
.../iceberg/spark/source/SparkInputPartition.java | 106 +
.../iceberg/spark/source/SparkLocalScan.java | 66 +
.../iceberg/spark/source/SparkMetadataColumn.java | 130 +
.../spark/source/SparkMicroBatchStream.java | 593 ++++
.../spark/source/SparkPartitioningAwareScan.java | 253 ++
.../iceberg/spark/source/SparkPlanningUtil.java | 93 +
.../spark/source/SparkPositionDeletesRewrite.java | 522 ++++
.../source/SparkPositionDeletesRewriteBuilder.java | 108 +
.../spark/source/SparkPositionDeltaOperation.java | 125 +
.../spark/source/SparkPositionDeltaWrite.java | 937 ++++++
.../source/SparkPositionDeltaWriteBuilder.java | 135 +
.../source/SparkRowLevelOperationBuilder.java | 114 +
.../spark/source/SparkRowReaderFactory.java | 68 +
.../org/apache/iceberg/spark/source/SparkScan.java | 354 +++
.../iceberg/spark/source/SparkScanBuilder.java | 778 +++++
.../iceberg/spark/source/SparkStagedScan.java | 98 +
.../spark/source/SparkStagedScanBuilder.java | 92 +
.../apache/iceberg/spark/source/SparkTable.java | 502 ++++
.../org/apache/iceberg/spark/source/SparkView.java | 151 +
.../apache/iceberg/spark/source/SparkWrite.java | 874 ++++++
.../iceberg/spark/source/SparkWriteBuilder.java | 238 ++
.../iceberg/spark/source/StagedSparkTable.java | 41 +
.../org/apache/iceberg/spark/source/Stats.java | 52 +
.../iceberg/spark/source/StreamingOffset.java | 161 ++
.../iceberg/spark/source/StructInternalRow.java | 416 +++
.../spark/source/metrics/EqualityDeleteFiles.java | 36 +
.../spark/source/metrics/IndexedDeleteFiles.java | 36 +
.../iceberg/spark/source/metrics/NumDeletes.java | 48 +
.../iceberg/spark/source/metrics/NumSplits.java | 46 +
.../source/metrics/PositionalDeleteFiles.java | 36 +
.../spark/source/metrics/ResultDataFiles.java | 36 +
.../spark/source/metrics/ResultDeleteFiles.java | 36 +
.../spark/source/metrics/ScannedDataManifests.java | 36 +
.../source/metrics/ScannedDeleteManifests.java | 36 +
.../spark/source/metrics/SkippedDataFiles.java | 36 +
.../spark/source/metrics/SkippedDataManifests.java | 36 +
.../spark/source/metrics/SkippedDeleteFiles.java | 36 +
.../source/metrics/SkippedDeleteManifests.java | 36 +
.../source/metrics/TaskEqualityDeleteFiles.java | 47 +
.../source/metrics/TaskIndexedDeleteFiles.java | 47 +
.../spark/source/metrics/TaskNumDeletes.java | 39 +
.../spark/source/metrics/TaskNumSplits.java | 39 +
.../source/metrics/TaskPositionalDeleteFiles.java | 47 +
.../spark/source/metrics/TaskResultDataFiles.java | 47 +
.../source/metrics/TaskResultDeleteFiles.java | 47 +
.../source/metrics/TaskScannedDataManifests.java | 47 +
.../source/metrics/TaskScannedDeleteManifests.java | 47 +
.../spark/source/metrics/TaskSkippedDataFiles.java | 47 +
.../source/metrics/TaskSkippedDataManifests.java | 47 +
.../source/metrics/TaskSkippedDeleteFiles.java | 47 +
.../source/metrics/TaskSkippedDeleteManifests.java | 47 +
.../source/metrics/TaskTotalDataFileSize.java | 48 +
.../source/metrics/TaskTotalDataManifests.java | 47 +
.../source/metrics/TaskTotalDeleteFileSize.java | 48 +
.../source/metrics/TaskTotalDeleteManifests.java | 47 +
.../source/metrics/TaskTotalPlanningDuration.java | 48 +
.../spark/source/metrics/TotalDataFileSize.java | 36 +
.../spark/source/metrics/TotalDataManifests.java | 36 +
.../spark/source/metrics/TotalDeleteFileSize.java | 36 +
.../spark/source/metrics/TotalDeleteManifests.java | 36 +
.../source/metrics/TotalPlanningDuration.java | 36 +
.../analysis/IcebergAnalysisException.java | 37 +
...org.apache.spark.sql.sources.DataSourceRegister | 20 +
.../plans/logical/OrderAwareCoalesce.scala | 48 +
.../logical/SetWriteDistributionAndOrdering.scala | 46 +
.../plans/logical/SortOrderParserUtil.scala | 41 +
.../spark/sql/catalyst/utils/PlanUtils.scala | 41 +
.../sql/execution/OrderAwareCoalesceExec.scala | 59 +
.../datasources/SparkExpressionConverter.scala | 73 +
.../apache/spark/sql/stats/ThetaSketchAgg.scala | 122 +
.../test/java/org/apache/iceberg/KryoHelpers.java | 51 +
.../iceberg/SparkDistributedDataScanTestBase.java | 95 +
.../java/org/apache/iceberg/TaskCheckHelper.java | 108 +
.../apache/iceberg/TestDataFileSerialization.java | 175 ++
.../apache/iceberg/TestFileIOSerialization.java | 106 +
.../TestHadoopMetricsContextSerialization.java | 56 +
.../iceberg/TestManifestFileSerialization.java | 221 ++
.../apache/iceberg/TestScanTaskSerialization.java | 197 ++
.../TestSparkDistributedDataScanDeletes.java | 91 +
.../TestSparkDistributedDataScanFilterFiles.java | 85 +
...tSparkDistributedDataScanJavaSerialization.java | 40 +
...tSparkDistributedDataScanKryoSerialization.java | 40 +
.../TestSparkDistributedDataScanReporting.java | 86 +
.../org/apache/iceberg/TestTableSerialization.java | 216 ++
.../java/org/apache/iceberg/ValidationHelpers.java | 78 +
.../org/apache/iceberg/spark/CatalogTestBase.java | 59 +
.../java/org/apache/iceberg/spark/Employee.java | 66 +
.../apache/iceberg/spark/SparkCatalogConfig.java | 87 +
.../apache/iceberg/spark/SparkTestHelperBase.java | 95 +
.../spark/SystemFunctionPushDownHelper.java | 127 +
.../java/org/apache/iceberg/spark/TestBase.java | 289 ++
.../apache/iceberg/spark/TestBaseWithCatalog.java | 196 ++
.../iceberg/spark/TestChangelogIterator.java | 359 +++
.../iceberg/spark/TestFileRewriteCoordinator.java | 275 ++
.../apache/iceberg/spark/TestFunctionCatalog.java | 136 +
.../org/apache/iceberg/spark/TestSpark3Util.java | 186 ++
.../iceberg/spark/TestSparkCachedTableCatalog.java | 105 +
.../iceberg/spark/TestSparkCatalogOperations.java | 196 ++
.../iceberg/spark/TestSparkCompressionUtil.java | 162 ++
.../TestSparkDistributionAndOrderingUtil.java | 3029 +++++++++++++++++++
.../iceberg/spark/TestSparkExecutorCache.java | 677 +++++
.../org/apache/iceberg/spark/TestSparkFilters.java | 201 ++
.../apache/iceberg/spark/TestSparkSchemaUtil.java | 272 ++
.../iceberg/spark/TestSparkSessionCatalog.java | 101 +
.../apache/iceberg/spark/TestSparkTableUtil.java | 106 +
.../apache/iceberg/spark/TestSparkV2Filters.java | 853 ++++++
.../iceberg/spark/TestSparkValueConverter.java | 94 +
.../apache/iceberg/spark/TestSparkWriteConf.java | 643 +++++
.../actions/TestComputePartitionStatsAction.java | 309 ++
.../spark/actions/TestComputeTableStatsAction.java | 443 +++
.../iceberg/spark/actions/TestCreateActions.java | 1056 +++++++
.../actions/TestDeleteReachableFilesAction.java | 403 +++
.../spark/actions/TestExpireSnapshotsAction.java | 1407 +++++++++
.../spark/actions/TestMigrateTableAction.java | 68 +
.../actions/TestRemoveDanglingDeleteAction.java | 540 ++++
.../spark/actions/TestRemoveOrphanFilesAction.java | 1308 +++++++++
.../actions/TestRemoveOrphanFilesAction3.java | 194 ++
.../spark/actions/TestRewriteDataFilesAction.java | 2660 +++++++++++++++++
.../spark/actions/TestRewriteManifestsAction.java | 1312 +++++++++
.../TestRewritePositionDeleteFilesAction.java | 1307 +++++++++
.../spark/actions/TestRewriteTablePathsAction.java | 1600 ++++++++++
.../spark/actions/TestSnapshotTableAction.java | 68 +
.../spark/actions/TestSparkFileRewriteRunners.java | 139 +
.../TestSparkShufflingDataRewritePlanner.java | 93 +
.../iceberg/spark/data/AvroDataTestBase.java | 728 +++++
.../apache/iceberg/spark/data/GenericsHelpers.java | 441 +++
.../org/apache/iceberg/spark/data/RandomData.java | 404 +++
.../org/apache/iceberg/spark/data/TestHelpers.java | 957 ++++++
.../apache/iceberg/spark/data/TestOrcWrite.java | 59 +
.../iceberg/spark/data/TestParquetAvroReader.java | 237 ++
.../iceberg/spark/data/TestParquetAvroWriter.java | 123 +
.../iceberg/spark/data/TestSparkAvroEnums.java | 97 +
.../iceberg/spark/data/TestSparkAvroReader.java | 91 +
.../iceberg/spark/data/TestSparkDateTimes.java | 78 +
.../data/TestSparkOrcReadMetadataColumns.java | 255 ++
.../iceberg/spark/data/TestSparkOrcReader.java | 127 +
.../data/TestSparkParquetReadMetadataColumns.java | 283 ++
.../iceberg/spark/data/TestSparkParquetReader.java | 263 ++
.../iceberg/spark/data/TestSparkParquetWriter.java | 182 ++
.../spark/data/TestSparkRecordOrcReaderWriter.java | 170 ++
.../iceberg/spark/data/TestSparkVariants.java | 344 +++
.../spark/data/TestVectorizedOrcDataReader.java | 167 ++
.../data/vectorized/TestColumnarBatchUtil.java | 302 ++
...estParquetDictionaryEncodedVectorizedReads.java | 199 ++
...naryFallbackToPlainEncodingVectorizedReads.java | 77 +
.../parquet/TestParquetVectorizedReads.java | 548 ++++
.../spark/functions/TestSparkFunctions.java | 158 +
.../iceberg/spark/source/BatchReaderUtil.java | 34 +
.../apache/iceberg/spark/source/ComplexRecord.java | 74 +
.../spark/source/DataFrameWriteTestBase.java | 149 +
.../spark/source/FilePathLastModifiedRecord.java | 80 +
.../iceberg/spark/source/FourColumnRecord.java | 106 +
.../apache/iceberg/spark/source/LogMessage.java | 119 +
.../apache/iceberg/spark/source/ManualSource.java | 75 +
.../apache/iceberg/spark/source/NestedRecord.java | 77 +
.../apache/iceberg/spark/source/ScanTestBase.java | 138 +
.../apache/iceberg/spark/source/SimpleRecord.java | 78 +
.../spark/source/SparkSQLExecutionHelper.java | 70 +
.../spark/source/TestAvroDataFrameWrite.java | 38 +
.../apache/iceberg/spark/source/TestAvroScan.java | 68 +
.../iceberg/spark/source/TestBaseReader.java | 289 ++
.../iceberg/spark/source/TestChangelogReader.java | 263 ++
.../spark/source/TestCompressionSettings.java | 276 ++
.../spark/source/TestDataFrameWriterV2.java | 390 +++
.../source/TestDataFrameWriterV2Coercion.java | 81 +
.../spark/source/TestDataSourceOptions.java | 520 ++++
.../iceberg/spark/source/TestFilteredScan.java | 760 +++++
.../spark/source/TestForwardCompatibility.java | 229 ++
.../iceberg/spark/source/TestIcebergSource.java | 42 +
.../source/TestIcebergSourceHadoopTables.java | 68 +
.../spark/source/TestIcebergSourceHiveTables.java | 86 +
.../spark/source/TestIcebergSourceTablesBase.java | 2459 ++++++++++++++++
.../iceberg/spark/source/TestIcebergSpark.java | 295 ++
.../spark/source/TestIdentityPartitionData.java | 251 ++
.../spark/source/TestInternalRowWrapper.java | 81 +
.../source/TestMetadataTableReadableMetrics.java | 379 +++
.../TestMetadataTablesWithPartitionEvolution.java | 725 +++++
.../spark/source/TestORCDataFrameWrite.java | 57 +
.../source/TestParquetCometVectorizedScan.java | 33 +
.../spark/source/TestParquetDataFrameWrite.java | 62 +
.../iceberg/spark/source/TestParquetScan.java | 113 +
.../spark/source/TestParquetVectorizedScan.java | 26 +
.../iceberg/spark/source/TestPartitionPruning.java | 489 ++++
.../iceberg/spark/source/TestPartitionValues.java | 511 ++++
.../iceberg/spark/source/TestPathIdentifier.java | 83 +
.../spark/source/TestPositionDeletesReader.java | 332 +++
.../spark/source/TestPositionDeletesTable.java | 1779 ++++++++++++
.../iceberg/spark/source/TestReadProjection.java | 654 +++++
.../TestRequiredDistributionAndOrdering.java | 310 ++
.../iceberg/spark/source/TestRuntimeFiltering.java | 510 ++++
.../spark/source/TestSnapshotSelection.java | 590 ++++
.../iceberg/spark/source/TestSparkAggregates.java | 76 +
.../iceberg/spark/source/TestSparkCatalog.java | 68 +
.../source/TestSparkCatalogCacheExpiration.java | 155 +
.../source/TestSparkCatalogHadoopOverrides.java | 143 +
.../iceberg/spark/source/TestSparkDVWriters.java | 69 +
.../iceberg/spark/source/TestSparkDataFile.java | 350 +++
.../iceberg/spark/source/TestSparkDataWrite.java | 739 +++++
.../spark/source/TestSparkFileWriterFactory.java | 69 +
.../spark/source/TestSparkMetadataColumns.java | 371 +++
.../spark/source/TestSparkPartitioningWriters.java | 69 +
.../spark/source/TestSparkPlanningUtil.java | 211 ++
.../source/TestSparkPositionDeltaWriters.java | 69 +
.../iceberg/spark/source/TestSparkReadMetrics.java | 279 ++
.../spark/source/TestSparkReadProjection.java | 273 ++
.../spark/source/TestSparkReaderDeletes.java | 755 +++++
.../source/TestSparkReaderWithBloomFilter.java | 376 +++
.../spark/source/TestSparkRollingFileWriters.java | 55 +
.../apache/iceberg/spark/source/TestSparkScan.java | 1089 +++++++
.../iceberg/spark/source/TestSparkStagedScan.java | 128 +
.../iceberg/spark/source/TestSparkTable.java | 59 +
.../spark/source/TestSparkWriterMetrics.java | 67 +
.../iceberg/spark/source/TestStreamingOffset.java | 56 +
.../spark/source/TestStructInternalRowVariant.java | 182 ++
.../spark/source/TestStructuredStreaming.java | 309 ++
.../spark/source/TestStructuredStreamingRead3.java | 942 ++++++
.../apache/iceberg/spark/source/TestTables.java | 209 ++
.../spark/source/TestTimestampWithoutZone.java | 229 ++
.../spark/source/TestWriteMetricsConfig.java | 304 ++
.../iceberg/spark/source/ThreeColumnRecord.java | 83 +
.../spark/sql/PartitionedWritesTestBase.java | 282 ++
.../iceberg/spark/sql/TestAggregatePushDown.java | 866 ++++++
.../apache/iceberg/spark/sql/TestAlterTable.java | 333 +++
.../iceberg/spark/sql/TestCTASEncryption.java | 120 +
.../apache/iceberg/spark/sql/TestCreateTable.java | 457 +++
.../iceberg/spark/sql/TestCreateTableAsSelect.java | 451 +++
.../apache/iceberg/spark/sql/TestDeleteFrom.java | 165 ++
.../apache/iceberg/spark/sql/TestDropTable.java | 158 +
.../iceberg/spark/sql/TestFilterPushDown.java | 703 +++++
.../apache/iceberg/spark/sql/TestNamespaceSQL.java | 302 ++
.../iceberg/spark/sql/TestPartitionedWrites.java | 21 +
.../spark/sql/TestPartitionedWritesAsSelect.java | 158 +
.../spark/sql/TestPartitionedWritesToBranch.java | 49 +
.../sql/TestPartitionedWritesToWapBranch.java | 91 +
.../apache/iceberg/spark/sql/TestRefreshTable.java | 86 +
.../org/apache/iceberg/spark/sql/TestSelect.java | 734 +++++
.../iceberg/spark/sql/TestSparkBucketFunction.java | 341 +++
.../iceberg/spark/sql/TestSparkDaysFunction.java | 107 +
.../iceberg/spark/sql/TestSparkDefaultValues.java | 234 ++
.../iceberg/spark/sql/TestSparkHoursFunction.java | 93 +
.../iceberg/spark/sql/TestSparkMonthsFunction.java | 125 +
.../spark/sql/TestSparkTruncateFunction.java | 443 +++
.../iceberg/spark/sql/TestSparkVariantRead.java | 316 ++
.../iceberg/spark/sql/TestSparkYearsFunction.java | 125 +
.../spark/sql/TestStoragePartitionedJoins.java | 881 ++++++
.../iceberg/spark/sql/TestTableEncryption.java | 341 +++
.../spark/sql/TestTimestampWithoutZone.java | 243 ++
.../iceberg/spark/sql/TestUnpartitionedWrites.java | 21 +
.../spark/sql/TestUnpartitionedWritesToBranch.java | 61 +
.../spark/sql/UnpartitionedWritesTestBase.java | 191 ++
.../decimal_dict_and_plain_encoding.parquet | Bin 0 -> 3685 bytes
574 files changed, 140378 insertions(+)
diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle
new file mode 100644
index 0000000000..8ebed9bd43
--- /dev/null
+++ b/spark/v4.0/build.gradle
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+String sparkMajorVersion = '4.0'
+String scalaVersion = '2.13'
+
+JavaVersion javaVersion = JavaVersion.current()
+Boolean javaVersionSupported = javaVersion == JavaVersion.VERSION_17 || javaVersion == JavaVersion.VERSION_21
+if (!javaVersionSupported) {
+ logger.warn("Skip Spark 4.0 build which requires JDK 17 or 21 but was executed with JDK " + javaVersion)
+}
+
+def sparkProjects = [
+ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}"),
+ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}"),
+ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}"),
+]
+
+configure(sparkProjects) {
+ tasks.configureEach {
+ onlyIf { javaVersionSupported }
+ }
+ configurations {
+ all {
+ resolutionStrategy {
+ force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson215.get()}"
+ force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson215.get()}"
+ force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson215.get()}"
+ }
+ }
+ }
+}
+
+project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
+ apply plugin: 'scala'
+ apply plugin: 'com.github.alisiikh.scalastyle'
+
+ sourceSets {
+ main {
+ scala.srcDirs = ['src/main/scala', 'src/main/java']
+ java.srcDirs = []
+ }
+ }
+
+ dependencies {
+ implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+ api project(':iceberg-api')
+ annotationProcessor libs.immutables.value
+ compileOnly libs.immutables.value
+ implementation project(':iceberg-common')
+ implementation project(':iceberg-core')
+ implementation project(':iceberg-data')
+ implementation project(':iceberg-orc')
+ implementation project(':iceberg-parquet')
+ implementation project(':iceberg-arrow')
+ implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
+ implementation("org.apache.datasketches:datasketches-java:${libs.versions.datasketches.get()}")
+
+ compileOnly libs.errorprone.annotations
+ compileOnly libs.avro.avro
+ compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark40.get()}") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
+ // to make sure netty libs only come from project(':iceberg-arrow')
+ exclude group: 'io.netty', module: 'netty-buffer'
+ exclude group: 'io.netty', module: 'netty-common'
+ exclude group: 'org.roaringbitmap'
+ }
+
+ compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_2.13:${libs.versions.comet.get()}"
+
+ implementation libs.parquet.column
+ implementation libs.parquet.hadoop
+
+ implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
+ exclude group: 'org.apache.hadoop'
+ exclude group: 'commons-lang'
+ // These artifacts are shaded and included in the orc-core fat jar
+ exclude group: 'com.google.protobuf', module: 'protobuf-java'
+ exclude group: 'org.apache.hive', module: 'hive-storage-api'
+ }
+
+ implementation(libs.arrow.vector) {
+ exclude group: 'io.netty', module: 'netty-buffer'
+ exclude group: 'io.netty', module: 'netty-common'
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+
+ implementation libs.caffeine
+
+ testImplementation(libs.hadoop3.minicluster) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ // to make sure netty libs only come from project(':iceberg-arrow')
+ exclude group: 'io.netty', module: 'netty-buffer'
+ exclude group: 'io.netty', module: 'netty-common'
+ }
+ testImplementation project(path: ':iceberg-hive-metastore')
+ testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
+ testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
+ transitive = false
+ }
+ testImplementation libs.sqlite.jdbc
+ testImplementation libs.awaitility
+ testImplementation(testFixtures(project(':iceberg-parquet')))
+ // runtime dependencies for running REST Catalog based integration test
+ testRuntimeOnly libs.jetty.servlet
+ }
+
+ test {
+ useJUnitPlatform()
+ }
+
+ tasks.withType(Test) {
+ // Vectorized reads need more memory
+ maxHeapSize '3160m'
+ }
+}
+
+project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") {
+ apply plugin: 'java-library'
+ apply plugin: 'scala'
+ apply plugin: 'com.github.alisiikh.scalastyle'
+ apply plugin: 'antlr'
+
+ configurations {
+ /*
+ The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This
+ bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate
+ runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with
+ the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add
+ the runtime dependency back to only the runtime configuration manually.
+ */
+ implementation {
+ extendsFrom = extendsFrom.findAll { it != configurations.antlr }
+ }
+ }
+
+ dependencies {
+ implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
+ implementation libs.roaringbitmap
+
+ compileOnly "org.scala-lang:scala-library"
+ compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+ compileOnly project(':iceberg-api')
+ compileOnly project(':iceberg-core')
+ compileOnly project(':iceberg-common')
+ compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
+ compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark40.get()}") {
+ exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.apache.arrow'
+ exclude group: 'org.apache.parquet'
+ // to make sure netty libs only come from project(':iceberg-arrow')
+ exclude group: 'io.netty', module: 'netty-buffer'
+ exclude group: 'io.netty', module: 'netty-common'
+ exclude group: 'org.roaringbitmap'
+ }
+ compileOnly libs.errorprone.annotations
+
+ testImplementation project(path: ':iceberg-data')
+ testImplementation project(path: ':iceberg-parquet')
+ testImplementation project(path: ':iceberg-hive-metastore')
+ testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+ testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
+ transitive = false
+ }
+ // runtime dependencies for running REST Catalog based integration test
+ testRuntimeOnly libs.jetty.servlet
+ testRuntimeOnly libs.sqlite.jdbc
+
+ testImplementation libs.avro.avro
+ testImplementation libs.parquet.hadoop
+ testImplementation libs.awaitility
+ testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_2.13:${libs.versions.comet.get()}"
+ testImplementation(testFixtures(project(':iceberg-parquet')))
+
+ // Required because we remove antlr plugin dependencies from the compile configuration, see note above
+ runtimeOnly libs.antlr.runtime413
+ antlr libs.antlr.antlr413
+ }
+
+ test {
+ useJUnitPlatform()
+ }
+
+ generateGrammarSource {
+ maxHeapSize = "64m"
+ arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions']
+ }
+}
+
+project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}") {
+ apply plugin: 'com.gradleup.shadow'
+
+ tasks.jar.dependsOn tasks.shadowJar
+
+ sourceSets {
+ integration {
+ java.srcDir "$projectDir/src/integration/java"
+ resources.srcDir "$projectDir/src/integration/resources"
+ }
+ }
+
+ configurations {
+ implementation {
+ exclude group: 'org.apache.spark'
+ // included in Spark
+ exclude group: 'org.slf4j'
+ exclude group: 'org.apache.commons'
+ exclude group: 'commons-pool'
+ exclude group: 'commons-codec'
+ exclude group: 'org.xerial.snappy'
+ exclude group: 'javax.xml.bind'
+ exclude group: 'javax.annotation'
+ exclude group: 'com.github.luben'
+ exclude group: 'com.ibm.icu'
+ exclude group: 'org.glassfish'
+ exclude group: 'org.abego.treelayout'
+ exclude group: 'org.antlr'
+ exclude group: 'org.scala-lang'
+ exclude group: 'org.scala-lang.modules'
+ }
+ }
+
+ dependencies {
+ api project(':iceberg-api')
+ implementation project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
+ implementation project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
+ implementation project(':iceberg-aws')
+ implementation project(':iceberg-azure')
+ implementation(project(':iceberg-aliyun')) {
+ exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
+ exclude group: 'org.apache.httpcomponents', module: 'httpclient'
+ exclude group: 'commons-logging', module: 'commons-logging'
+ }
+ implementation project(':iceberg-gcp')
+ implementation project(':iceberg-bigquery')
+ implementation project(':iceberg-hive-metastore')
+ implementation(project(':iceberg-nessie')) {
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+ implementation (project(':iceberg-snowflake')) {
+ exclude group: 'net.snowflake' , module: 'snowflake-jdbc'
+ }
+
+ integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
+ integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark40.get()}"
+ integrationImplementation libs.junit.jupiter
+ integrationImplementation libs.junit.platform.launcher
+ integrationImplementation libs.slf4j.simple
+ integrationImplementation libs.assertj.core
+ integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
+ integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+ integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+
+ // runtime dependencies for running Hive Catalog based integration test
+ integrationRuntimeOnly project(':iceberg-hive-metastore')
+ // runtime dependencies for running REST Catalog based integration test
+ integrationRuntimeOnly project(path: ':iceberg-core', configuration: 'testArtifacts')
+ integrationRuntimeOnly (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
+ transitive = false
+ }
+ integrationRuntimeOnly libs.jetty.servlet
+ integrationRuntimeOnly libs.sqlite.jdbc
+
+ // Not allowed on our classpath, only the runtime jar is allowed
+ integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
+ integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
+ integrationCompileOnly project(':iceberg-api')
+ }
+
+ shadowJar {
+ configurations = [project.configurations.runtimeClasspath]
+
+ zip64 true
+
+ // include the LICENSE and NOTICE files for the shaded Jar
+ from(projectDir) {
+ include 'LICENSE'
+ include 'NOTICE'
+ }
+
+ // Relocate dependencies to avoid conflicts
+ relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
+ relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
+ relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
+ relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
+ relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
+ relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
+ relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
+ relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
+ relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
+ relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
+ relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
+ relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
+ relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
+ relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
+ // relocate Arrow and related deps to shade Iceberg specific version
+ relocate 'io.netty', 'org.apache.iceberg.shaded.io.netty'
+ relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow'
+ relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch'
+ relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
+ relocate 'org.roaringbitmap', 'org.apache.iceberg.shaded.org.roaringbitmap'
+ relocate 'org.apache.datasketches', 'org.apache.iceberg.shaded.org.apache.datasketches'
+
+ archiveClassifier.set(null)
+ }
+
+ task integrationTest(type: Test) {
+ useJUnitPlatform()
+ description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
+ group = "verification"
+ jvmArgs += project.property('extraJvmArgs')
+ testClassesDirs = sourceSets.integration.output.classesDirs
+ classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
+ inputs.file(shadowJar.archiveFile.get().asFile.path)
+ }
+ integrationTest.dependsOn shadowJar
+ check.dependsOn integrationTest
+
+ jar {
+ enabled = false
+ }
+}
+
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/DeleteFileIndexBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/DeleteFileIndexBenchmark.java
new file mode 100644
index 0000000000..9375ca3a4f
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/DeleteFileIndexBenchmark.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import com.google.errorprone.annotations.FormatMethod;
+import com.google.errorprone.annotations.FormatString;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.Spark3Util;
+import org.apache.iceberg.spark.SparkSessionCatalog;
+import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
+import org.apache.iceberg.util.ThreadPools;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.parser.ParseException;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Timeout;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * A benchmark that evaluates the delete file index build and lookup performance.
+ *
+ * <p>To run this benchmark for spark-4.0: <code>
+ * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.13:jmh
+ * -PjmhIncludeRegex=DeleteFileIndexBenchmark
+ * -PjmhOutputPath=benchmark/iceberg-delete-file-index-benchmark.txt
+ * </code>
+ */
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 10)
+@Timeout(time = 20, timeUnit = TimeUnit.MINUTES)
+@BenchmarkMode(Mode.SingleShotTime)
+public class DeleteFileIndexBenchmark {
+
+ private static final String TABLE_NAME = "test_table";
+ private static final String PARTITION_COLUMN = "ss_ticket_number";
+
+ private static final int NUM_PARTITIONS = 50;
+ private static final int NUM_DATA_FILES_PER_PARTITION = 50_000;
+ private static final int NUM_DELETE_FILES_PER_PARTITION = 100;
+
+ private final Configuration hadoopConf = new Configuration();
+ private SparkSession spark;
+ private Table table;
+
+ private List<DataFile> dataFiles;
+
+ @Param({"partition", "file", "dv"})
+ private String type;
+
+ @Setup
+ public void setupBenchmark() throws NoSuchTableException, ParseException {
+ setupSpark();
+ initTable();
+ initDataAndDeletes();
+ loadDataFiles();
+ }
+
+ private void initDataAndDeletes() {
+ if (type.equals("partition")) {
+ initDataAndPartitionScopedDeletes();
+ } else if (type.equals("file")) {
+ initDataAndFileScopedDeletes();
+ } else {
+ initDataAndDVs();
+ }
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {
+ dropTable();
+ tearDownSpark();
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void buildIndexAndLookup(Blackhole blackhole) {
+ DeleteFileIndex deletes = buildDeletes();
+ for (DataFile dataFile : dataFiles) {
+ DeleteFile[] deleteFiles = deletes.forDataFile(dataFile.dataSequenceNumber(), dataFile);
+ blackhole.consume(deleteFiles);
+ }
+ }
+
+ private void loadDataFiles() {
+ table.refresh();
+
+ Snapshot snapshot = table.currentSnapshot();
+
+ ManifestGroup manifestGroup =
+ new ManifestGroup(table.io(), snapshot.dataManifests(table.io()), ImmutableList.of());
+
+ try (CloseableIterable<ManifestEntry<DataFile>> entries = manifestGroup.entries()) {
+ List<DataFile> files = Lists.newArrayList();
+ for (ManifestEntry<DataFile> entry : entries) {
+ files.add(entry.file().copyWithoutStats());
+ }
+ this.dataFiles = files;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private DeleteFileIndex buildDeletes() {
+ table.refresh();
+
+ List<ManifestFile> deleteManifests = table.currentSnapshot().deleteManifests(table.io());
+
+ return DeleteFileIndex.builderFor(table.io(), deleteManifests)
+ .specsById(table.specs())
+ .planWith(ThreadPools.getWorkerPool())
+ .build();
+ }
+
+ private void initDataAndPartitionScopedDeletes() {
+ for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
+ StructLike partition = TestHelpers.Row.of(partitionOrdinal);
+
+ RowDelta rowDelta = table.newRowDelta();
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
+ DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
+ rowDelta.addRows(dataFile);
+ }
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DELETE_FILES_PER_PARTITION; fileOrdinal++) {
+ DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, partition);
+ rowDelta.addDeletes(deleteFile);
+ }
+
+ rowDelta.commit();
+ }
+ }
+
+ private void initDataAndFileScopedDeletes() {
+ for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
+ StructLike partition = TestHelpers.Row.of(partitionOrdinal);
+
+ RowDelta rowDelta = table.newRowDelta();
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
+ DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
+ DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
+ rowDelta.addRows(dataFile);
+ rowDelta.addDeletes(deleteFile);
+ }
+
+ rowDelta.commit();
+ }
+ }
+
+ private void initDataAndDVs() {
+ for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
+ StructLike partition = TestHelpers.Row.of(partitionOrdinal);
+
+ RowDelta rowDelta = table.newRowDelta();
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
+ DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
+ DeleteFile dv = FileGenerationUtil.generateDV(table, dataFile);
+ rowDelta.addRows(dataFile);
+ rowDelta.addDeletes(dv);
+ }
+
+ rowDelta.commit();
+ }
+ }
+
+ private void setupSpark() {
+ this.spark =
+ SparkSession.builder()
+ .config("spark.ui.enabled", false)
+ .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.type", "hadoop")
+ .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
+ .master("local[*]")
+ .getOrCreate();
+ }
+
+ private void tearDownSpark() {
+ spark.stop();
+ }
+
+ private void initTable() throws NoSuchTableException, ParseException {
+ sql(
+ "CREATE TABLE %s ( "
+ + " `ss_sold_date_sk` INT, "
+ + " `ss_sold_time_sk` INT, "
+ + " `ss_item_sk` INT, "
+ + " `ss_customer_sk` STRING, "
+ + " `ss_cdemo_sk` STRING, "
+ + " `ss_hdemo_sk` STRING, "
+ + " `ss_addr_sk` STRING, "
+ + " `ss_store_sk` STRING, "
+ + " `ss_promo_sk` STRING, "
+ + " `ss_ticket_number` INT, "
+ + " `ss_quantity` STRING, "
+ + " `ss_wholesale_cost` STRING, "
+ + " `ss_list_price` STRING, "
+ + " `ss_sales_price` STRING, "
+ + " `ss_ext_discount_amt` STRING, "
+ + " `ss_ext_sales_price` STRING, "
+ + " `ss_ext_wholesale_cost` STRING, "
+ + " `ss_ext_list_price` STRING, "
+ + " `ss_ext_tax` STRING, "
+ + " `ss_coupon_amt` STRING, "
+ + " `ss_net_paid` STRING, "
+ + " `ss_net_paid_inc_tax` STRING, "
+ + " `ss_net_profit` STRING "
+ + ")"
+ + "USING iceberg "
+ + "PARTITIONED BY (%s) "
+ + "TBLPROPERTIES ("
+ + " '%s' '%b',"
+ + " '%s' '%s',"
+ + " '%s' '%d')",
+ TABLE_NAME,
+ PARTITION_COLUMN,
+ TableProperties.MANIFEST_MERGE_ENABLED,
+ false,
+ TableProperties.DELETE_MODE,
+ RowLevelOperationMode.MERGE_ON_READ.modeName(),
+ TableProperties.FORMAT_VERSION,
+ type.equals("dv") ? 3 : 2);
+
+ this.table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
+ }
+
+ private void dropTable() {
+ sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
+ }
+
+ private String newWarehouseDir() {
+ return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
+ }
+
+ @FormatMethod
+ private void sql(@FormatString String query, Object... args) {
+ spark.sql(String.format(query, args));
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
new file mode 100644
index 0000000000..963daa2c36
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.apache.spark.sql.functions.current_date;
+import static org.apache.spark.sql.functions.date_add;
+import static org.apache.spark.sql.functions.expr;
+
+import com.google.errorprone.annotations.FormatMethod;
+import com.google.errorprone.annotations.FormatString;
+import java.util.UUID;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.DistributionMode;
+import org.apache.iceberg.RowLevelOperationMode;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.parser.ParseException;
+import org.apache.spark.sql.internal.SQLConf;
+import org.apache.spark.sql.types.StructType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * A benchmark that evaluates the performance of the cardinality check in MERGE operations.
+ *
+ * <p>To run this benchmark for spark-4.0: <code>
+ * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.13:jmh
+ * -PjmhIncludeRegex=MergeCardinalityCheckBenchmark
+ * -PjmhOutputPath=benchmark/iceberg-merge-cardinality-check-benchmark.txt
+ * </code>
+ */
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.SingleShotTime)
+public class MergeCardinalityCheckBenchmark {
+
+ private static final String TABLE_NAME = "test_table";
+ private static final int NUM_FILES = 5;
+ private static final int NUM_ROWS_PER_FILE = 1_000_000;
+ private static final int NUM_UNMATCHED_RECORDS_PER_MERGE = 100_000;
+
+ private final Configuration hadoopConf = new Configuration();
+ private SparkSession spark;
+ private long originalSnapshotId;
+
+ @Setup
+ public void setupBenchmark() throws NoSuchTableException, ParseException {
+ setupSpark();
+ initTable();
+ appendData();
+
+ Table table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
+ this.originalSnapshotId = table.currentSnapshot().snapshotId();
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {
+ tearDownSpark();
+ dropTable();
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void copyOnWriteMergeCardinalityCheck10PercentUpdates() {
+ runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.1);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void copyOnWriteMergeCardinalityCheck30PercentUpdates() {
+ runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.3);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void copyOnWriteMergeCardinalityCheck90PercentUpdates() {
+ runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.9);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void mergeOnReadMergeCardinalityCheck10PercentUpdates() {
+ runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.1);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void mergeOnReadMergeCardinalityCheck30PercentUpdates() {
+ runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.3);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void mergeOnReadMergeCardinalityCheck90PercentUpdates() {
+ runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.9);
+ }
+
+ private void runBenchmark(RowLevelOperationMode mode, double updatePercentage) {
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')",
+ TABLE_NAME, TableProperties.MERGE_MODE, mode.modeName());
+
+ Dataset<Long> insertDataDF = spark.range(-NUM_UNMATCHED_RECORDS_PER_MERGE, 0, 1);
+ Dataset<Long> updateDataDF = spark.range((long) (updatePercentage * NUM_ROWS_PER_FILE));
+ Dataset<Long> sourceDF = updateDataDF.union(insertDataDF);
+ sourceDF.createOrReplaceTempView("source");
+
+ sql(
+ "MERGE INTO %s t USING source s "
+ + "ON t.id = s.id "
+ + "WHEN MATCHED THEN "
+ + " UPDATE SET stringCol = 'invalid' "
+ + "WHEN NOT MATCHED THEN "
+ + " INSERT (id, intCol, floatCol, doubleCol, decimalCol, dateCol, timestampCol, stringCol) "
+ + " VALUES (s.id, null, null, null, null, null, null, 'new')",
+ TABLE_NAME);
+
+ sql(
+ "CALL system.rollback_to_snapshot(table => '%s', snapshot_id => %dL)",
+ TABLE_NAME, originalSnapshotId);
+ }
+
+ private void setupSpark() {
+ this.spark =
+ SparkSession.builder()
+ .config("spark.ui.enabled", false)
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.type", "hadoop")
+ .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
+ .config(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key(), "false")
+ .config(SQLConf.RUNTIME_ROW_LEVEL_OPERATION_GROUP_FILTER_ENABLED().key(), "false")
+ .config(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false")
+ .config(SQLConf.SHUFFLE_PARTITIONS().key(), "2")
+ .master("local")
+ .getOrCreate();
+ }
+
+ private void tearDownSpark() {
+ spark.stop();
+ }
+
+ private void initTable() {
+ sql(
+ "CREATE TABLE %s ( "
+ + " id LONG, intCol INT, floatCol FLOAT, doubleCol DOUBLE, "
+ + " decimalCol DECIMAL(20, 5), dateCol DATE, timestampCol TIMESTAMP, "
+ + " stringCol STRING)"
+ + "USING iceberg "
+ + "TBLPROPERTIES ("
+ + " '%s' '%s',"
+ + " '%s' '%d',"
+ + " '%s' '%d')",
+ TABLE_NAME,
+ TableProperties.MERGE_DISTRIBUTION_MODE,
+ DistributionMode.NONE.modeName(),
+ TableProperties.SPLIT_OPEN_FILE_COST,
+ Integer.MAX_VALUE,
+ TableProperties.FORMAT_VERSION,
+ 2);
+
+ sql("ALTER TABLE %s WRITE ORDERED BY id", TABLE_NAME);
+ }
+
+ private void dropTable() {
+ sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
+ }
+
+ private void appendData() throws NoSuchTableException {
+ for (int fileNum = 1; fileNum <= NUM_FILES; fileNum++) {
+ Dataset<Row> inputDF =
+ spark
+ .range(NUM_ROWS_PER_FILE)
+ .withColumn("intCol", expr("CAST(id AS INT)"))
+ .withColumn("floatCol", expr("CAST(id AS FLOAT)"))
+ .withColumn("doubleCol", expr("CAST(id AS DOUBLE)"))
+ .withColumn("decimalCol", expr("CAST(id AS DECIMAL(20, 5))"))
+ .withColumn("dateCol", date_add(current_date(), fileNum))
+ .withColumn("timestampCol", expr("TO_TIMESTAMP(dateCol)"))
+ .withColumn("stringCol", expr("CAST(dateCol AS STRING)"));
+ appendAsFile(inputDF);
+ }
+ }
+
+ private void appendAsFile(Dataset<Row> df) throws NoSuchTableException {
+ // ensure the schema is precise (including nullability)
+ StructType sparkSchema = spark.table(TABLE_NAME).schema();
+ spark.createDataFrame(df.rdd(), sparkSchema).coalesce(1).writeTo(TABLE_NAME).append();
+ }
+
+ private String newWarehouseDir() {
+ return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
+ }
+
+ @FormatMethod
+ private void sql(@FormatString String query, Object... args) {
+ spark.sql(String.format(query, args));
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/PlanningBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/PlanningBenchmark.java
new file mode 100644
index 0000000000..34d9d70e6c
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/PlanningBenchmark.java
@@ -0,0 +1,409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.apache.iceberg.PlanningMode.DISTRIBUTED;
+import static org.apache.iceberg.PlanningMode.LOCAL;
+
+import com.google.errorprone.annotations.FormatMethod;
+import com.google.errorprone.annotations.FormatString;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BatchScan;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileGenerationUtil;
+import org.apache.iceberg.PlanningMode;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.RowLevelOperationMode;
+import org.apache.iceberg.ScanTask;
+import org.apache.iceberg.SparkDistributedDataScan;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
+import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.types.Types;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.parser.ParseException;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Timeout;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * A benchmark that evaluates the job planning performance.
+ *
+ * <p>To run this benchmark for spark-4.0: <code>
+ * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.12:jmh
+ * -PjmhIncludeRegex=PlanningBenchmark
+ * -PjmhOutputPath=benchmark/iceberg-planning-benchmark.txt
+ * </code>
+ */
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@Timeout(time = 20, timeUnit = TimeUnit.MINUTES)
+@BenchmarkMode(Mode.SingleShotTime)
+public class PlanningBenchmark {
+
+ private static final String TABLE_NAME = "test_table";
+ private static final String PARTITION_COLUMN = "ss_ticket_number";
+ private static final int PARTITION_VALUE = 10;
+ private static final String SORT_KEY_COLUMN = "ss_sold_date_sk";
+ private static final int SORT_KEY_VALUE = 5;
+
+ private static final Expression SORT_KEY_PREDICATE =
+ Expressions.equal(SORT_KEY_COLUMN, SORT_KEY_VALUE);
+ private static final Expression PARTITION_PREDICATE =
+ Expressions.equal(PARTITION_COLUMN, PARTITION_VALUE);
+ private static final Expression PARTITION_AND_SORT_KEY_PREDICATE =
+ Expressions.and(PARTITION_PREDICATE, SORT_KEY_PREDICATE);
+
+ private static final int NUM_PARTITIONS = 30;
+ private static final int NUM_DATA_FILES_PER_PARTITION = 50_000;
+ private static final int NUM_DELETE_FILES_PER_PARTITION = 50;
+
+ private final Configuration hadoopConf = new Configuration();
+ private SparkSession spark;
+ private Table table;
+
+ @Param({"partition", "file", "dv"})
+ private String type;
+
+ @Setup
+ public void setupBenchmark() throws NoSuchTableException, ParseException {
+ setupSpark();
+ initTable();
+ initDataAndDeletes();
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {
+ dropTable();
+ tearDownSpark();
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void localPlanningWithPartitionAndMinMaxFilter(Blackhole blackhole) {
+ BatchScan scan = table.newBatchScan();
+ List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, PARTITION_AND_SORT_KEY_PREDICATE);
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void distributedPlanningWithPartitionAndMinMaxFilter(Blackhole blackhole) {
+ BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
+ List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, PARTITION_AND_SORT_KEY_PREDICATE);
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void localPlanningWithMinMaxFilter(Blackhole blackhole) {
+ BatchScan scan = table.newBatchScan();
+ List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, SORT_KEY_PREDICATE);
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void distributedPlanningWithMinMaxFilter(Blackhole blackhole) {
+ BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
+ List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, SORT_KEY_PREDICATE);
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void localPlanningWithoutFilter(Blackhole blackhole) {
+ BatchScan scan = table.newBatchScan();
+ List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, Expressions.alwaysTrue());
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void distributedPlanningWithoutFilter(Blackhole blackhole) {
+ BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
+ List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, Expressions.alwaysTrue());
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void localPlanningWithoutFilterWithStats(Blackhole blackhole) {
+ BatchScan scan = table.newBatchScan();
+ List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void distributedPlanningWithoutFilterWithStats(Blackhole blackhole) {
+ BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
+ List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void distributedDataLocalDeletesPlanningWithoutFilterWithStats(Blackhole blackhole) {
+ BatchScan scan = newDistributedScan(DISTRIBUTED, LOCAL);
+ List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void localDataDistributedDeletesPlanningWithoutFilterWithStats(Blackhole blackhole) {
+ BatchScan scan = newDistributedScan(LOCAL, DISTRIBUTED);
+ List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
+ blackhole.consume(fileTasks);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void localPlanningViaDistributedScanWithoutFilterWithStats(Blackhole blackhole) {
+ BatchScan scan = newDistributedScan(LOCAL, LOCAL);
+ List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
+ blackhole.consume(fileTasks);
+ }
+
+ private void setupSpark() {
+ this.spark =
+ SparkSession.builder()
+ .config("spark.ui.enabled", false)
+ .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+ .config("spark.driver.maxResultSize", "8G")
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.type", "hadoop")
+ .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
+ .master("local[*]")
+ .getOrCreate();
+ }
+
+ private void tearDownSpark() {
+ spark.stop();
+ }
+
+ private void initTable() throws NoSuchTableException, ParseException {
+ sql(
+ "CREATE TABLE %s ( "
+ + " `ss_sold_date_sk` INT, "
+ + " `ss_sold_time_sk` INT, "
+ + " `ss_item_sk` INT, "
+ + " `ss_customer_sk` STRING, "
+ + " `ss_cdemo_sk` STRING, "
+ + " `ss_hdemo_sk` STRING, "
+ + " `ss_addr_sk` STRING, "
+ + " `ss_store_sk` STRING, "
+ + " `ss_promo_sk` STRING, "
+ + " `ss_ticket_number` INT, "
+ + " `ss_quantity` STRING, "
+ + " `ss_wholesale_cost` STRING, "
+ + " `ss_list_price` STRING, "
+ + " `ss_sales_price` STRING, "
+ + " `ss_ext_discount_amt` STRING, "
+ + " `ss_ext_sales_price` STRING, "
+ + " `ss_ext_wholesale_cost` STRING, "
+ + " `ss_ext_list_price` STRING, "
+ + " `ss_ext_tax` STRING, "
+ + " `ss_coupon_amt` STRING, "
+ + " `ss_net_paid` STRING, "
+ + " `ss_net_paid_inc_tax` STRING, "
+ + " `ss_net_profit` STRING "
+ + ")"
+ + "USING iceberg "
+ + "PARTITIONED BY (%s) "
+ + "TBLPROPERTIES ("
+ + " '%s' '%b',"
+ + " '%s' '%s',"
+ + " '%s' '%d')",
+ TABLE_NAME,
+ PARTITION_COLUMN,
+ TableProperties.MANIFEST_MERGE_ENABLED,
+ false,
+ TableProperties.DELETE_MODE,
+ RowLevelOperationMode.MERGE_ON_READ.modeName(),
+ TableProperties.FORMAT_VERSION,
+ type.equals("dv") ? 3 : 2);
+
+ this.table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
+ }
+
+ private void dropTable() {
+ sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
+ }
+
+ private void initDataAndDeletes() {
+ if (type.equals("partition")) {
+ initDataAndPartitionScopedDeletes();
+ } else if (type.equals("file")) {
+ initDataAndFileScopedDeletes();
+ } else {
+ initDataAndDVs();
+ }
+ }
+
+ private void initDataAndPartitionScopedDeletes() {
+ for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
+ StructLike partition = TestHelpers.Row.of(partitionOrdinal);
+
+ RowDelta rowDelta = table.newRowDelta();
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
+ DataFile dataFile = generateDataFile(partition, Integer.MIN_VALUE, Integer.MIN_VALUE);
+ rowDelta.addRows(dataFile);
+ }
+
+ // add one data file that would match the sort key predicate
+ DataFile sortKeyDataFile = generateDataFile(partition, SORT_KEY_VALUE, SORT_KEY_VALUE);
+ rowDelta.addRows(sortKeyDataFile);
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DELETE_FILES_PER_PARTITION; fileOrdinal++) {
+ DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, partition);
+ rowDelta.addDeletes(deleteFile);
+ }
+
+ rowDelta.commit();
+ }
+ }
+
+ private void initDataAndFileScopedDeletes() {
+ for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
+ StructLike partition = TestHelpers.Row.of(partitionOrdinal);
+
+ RowDelta rowDelta = table.newRowDelta();
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
+ DataFile dataFile = generateDataFile(partition, Integer.MIN_VALUE, Integer.MIN_VALUE);
+ DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
+ rowDelta.addRows(dataFile);
+ rowDelta.addDeletes(deleteFile);
+ }
+
+ // add one data file that would match the sort key predicate
+ DataFile sortKeyDataFile = generateDataFile(partition, SORT_KEY_VALUE, SORT_KEY_VALUE);
+ rowDelta.addRows(sortKeyDataFile);
+
+ rowDelta.commit();
+ }
+ }
+
+ private void initDataAndDVs() {
+ for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
+ StructLike partition = TestHelpers.Row.of(partitionOrdinal);
+
+ RowDelta rowDelta = table.newRowDelta();
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
+ DataFile dataFile = generateDataFile(partition, Integer.MIN_VALUE, Integer.MIN_VALUE);
+ DeleteFile dv = FileGenerationUtil.generateDV(table, dataFile);
+ rowDelta.addRows(dataFile);
+ rowDelta.addDeletes(dv);
+ }
+
+ // add one data file that would match the sort key predicate
+ DataFile sortKeyDataFile = generateDataFile(partition, SORT_KEY_VALUE, SORT_KEY_VALUE);
+ rowDelta.addRows(sortKeyDataFile);
+
+ rowDelta.commit();
+ }
+ }
+
+ private DataFile generateDataFile(StructLike partition, int sortKeyMin, int sortKeyMax) {
+ int sortKeyFieldId = table.schema().findField(SORT_KEY_COLUMN).fieldId();
+ ByteBuffer lower = Conversions.toByteBuffer(Types.IntegerType.get(), sortKeyMin);
+ Map<Integer, ByteBuffer> lowerBounds = ImmutableMap.of(sortKeyFieldId, lower);
+ ByteBuffer upper = Conversions.toByteBuffer(Types.IntegerType.get(), sortKeyMax);
+ Map<Integer, ByteBuffer> upperBounds = ImmutableMap.of(sortKeyFieldId, upper);
+ return FileGenerationUtil.generateDataFile(table, partition, lowerBounds, upperBounds);
+ }
+
+ private String newWarehouseDir() {
+ return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
+ }
+
+ private List<ScanTask> planFilesWithoutColumnStats(BatchScan scan, Expression predicate) {
+ return planFiles(scan, predicate, false);
+ }
+
+ private List<ScanTask> planFilesWithColumnStats(BatchScan scan, Expression predicate) {
+ return planFiles(scan, predicate, true);
+ }
+
+ private List<ScanTask> planFiles(BatchScan scan, Expression predicate, boolean withColumnStats) {
+ table.refresh();
+
+ BatchScan configuredScan = scan.filter(predicate);
+
+ if (withColumnStats) {
+ configuredScan = scan.includeColumnStats();
+ }
+
+ try (CloseableIterable<ScanTask> fileTasks = configuredScan.planFiles()) {
+ return Lists.newArrayList(fileTasks);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private BatchScan newDistributedScan(PlanningMode dataMode, PlanningMode deleteMode) {
+ table
+ .updateProperties()
+ .set(TableProperties.DATA_PLANNING_MODE, dataMode.modeName())
+ .set(TableProperties.DELETE_PLANNING_MODE, deleteMode.modeName())
+ .commit();
+ SparkReadConf readConf = new SparkReadConf(spark, table, ImmutableMap.of());
+ return new SparkDistributedDataScan(spark, table, readConf);
+ }
+
+ @FormatMethod
+ private void sql(@FormatString String query, Object... args) {
+ spark.sql(String.format(query, args));
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/TaskGroupPlanningBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/TaskGroupPlanningBenchmark.java
new file mode 100644
index 0000000000..7c2def2378
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/TaskGroupPlanningBenchmark.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import com.google.errorprone.annotations.FormatMethod;
+import com.google.errorprone.annotations.FormatString;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileGenerationUtil;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.RowLevelOperationMode;
+import org.apache.iceberg.ScanTaskGroup;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
+import org.apache.iceberg.util.TableScanUtil;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.parser.ParseException;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Timeout;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * A benchmark that evaluates the task group planning performance.
+ *
+ * <p>To run this benchmark for spark-4.0: <code>
+ * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.13:jmh
+ * -PjmhIncludeRegex=TaskGroupPlanningBenchmark
+ * -PjmhOutputPath=benchmark/iceberg-task-group-planning-benchmark.txt
+ * </code>
+ */
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@Timeout(time = 30, timeUnit = TimeUnit.MINUTES)
+@BenchmarkMode(Mode.SingleShotTime)
+public class TaskGroupPlanningBenchmark {
+
+ private static final String TABLE_NAME = "test_table";
+ private static final String PARTITION_COLUMN = "ss_ticket_number";
+
+ private static final int NUM_PARTITIONS = 150;
+ private static final int NUM_DATA_FILES_PER_PARTITION = 50_000;
+ private static final int NUM_DELETE_FILES_PER_PARTITION = 25;
+
+ private final Configuration hadoopConf = new Configuration();
+ private SparkSession spark;
+ private Table table;
+
+ private List<FileScanTask> fileTasks;
+
+ @Setup
+ public void setupBenchmark() throws NoSuchTableException, ParseException {
+ setupSpark();
+ initTable();
+ initDataAndDeletes();
+ loadFileTasks();
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {
+ dropTable();
+ tearDownSpark();
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void planTaskGroups(Blackhole blackhole) {
+ SparkReadConf readConf = new SparkReadConf(spark, table, ImmutableMap.of());
+ List<ScanTaskGroup<FileScanTask>> taskGroups =
+ TableScanUtil.planTaskGroups(
+ fileTasks,
+ readConf.splitSize(),
+ readConf.splitLookback(),
+ readConf.splitOpenFileCost());
+
+ long rowsCount = 0L;
+ for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
+ rowsCount += taskGroup.estimatedRowsCount();
+ }
+ blackhole.consume(rowsCount);
+
+ long filesCount = 0L;
+ for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
+ filesCount += taskGroup.filesCount();
+ }
+ blackhole.consume(filesCount);
+
+ long sizeBytes = 0L;
+ for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
+ sizeBytes += taskGroup.sizeBytes();
+ }
+ blackhole.consume(sizeBytes);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void planTaskGroupsWithGrouping(Blackhole blackhole) {
+ SparkReadConf readConf = new SparkReadConf(spark, table, ImmutableMap.of());
+
+ List<ScanTaskGroup<FileScanTask>> taskGroups =
+ TableScanUtil.planTaskGroups(
+ fileTasks,
+ readConf.splitSize(),
+ readConf.splitLookback(),
+ readConf.splitOpenFileCost(),
+ Partitioning.groupingKeyType(table.schema(), table.specs().values()));
+
+ long rowsCount = 0L;
+ for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
+ rowsCount += taskGroup.estimatedRowsCount();
+ }
+ blackhole.consume(rowsCount);
+
+ long filesCount = 0L;
+ for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
+ filesCount += taskGroup.filesCount();
+ }
+ blackhole.consume(filesCount);
+
+ long sizeBytes = 0L;
+ for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
+ sizeBytes += taskGroup.sizeBytes();
+ }
+ blackhole.consume(sizeBytes);
+ }
+
+ private void loadFileTasks() {
+ table.refresh();
+
+ try (CloseableIterable<FileScanTask> fileTasksIterable = table.newScan().planFiles()) {
+ this.fileTasks = Lists.newArrayList(fileTasksIterable);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private void initDataAndDeletes() {
+ for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
+ StructLike partition = TestHelpers.Row.of(partitionOrdinal);
+
+ RowDelta rowDelta = table.newRowDelta();
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
+ DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
+ rowDelta.addRows(dataFile);
+ }
+
+ for (int fileOrdinal = 0; fileOrdinal < NUM_DELETE_FILES_PER_PARTITION; fileOrdinal++) {
+ DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, partition);
+ rowDelta.addDeletes(deleteFile);
+ }
+
+ rowDelta.commit();
+ }
+ }
+
+ private void setupSpark() {
+ this.spark =
+ SparkSession.builder()
+ .config("spark.ui.enabled", false)
+ .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.type", "hadoop")
+ .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
+ .master("local[*]")
+ .getOrCreate();
+ }
+
+ private void tearDownSpark() {
+ spark.stop();
+ }
+
+ private void initTable() throws NoSuchTableException, ParseException {
+ sql(
+ "CREATE TABLE %s ( "
+ + " `ss_sold_date_sk` INT, "
+ + " `ss_sold_time_sk` INT, "
+ + " `ss_item_sk` INT, "
+ + " `ss_customer_sk` STRING, "
+ + " `ss_cdemo_sk` STRING, "
+ + " `ss_hdemo_sk` STRING, "
+ + " `ss_addr_sk` STRING, "
+ + " `ss_store_sk` STRING, "
+ + " `ss_promo_sk` STRING, "
+ + " `ss_ticket_number` INT, "
+ + " `ss_quantity` STRING, "
+ + " `ss_wholesale_cost` STRING, "
+ + " `ss_list_price` STRING, "
+ + " `ss_sales_price` STRING, "
+ + " `ss_ext_discount_amt` STRING, "
+ + " `ss_ext_sales_price` STRING, "
+ + " `ss_ext_wholesale_cost` STRING, "
+ + " `ss_ext_list_price` STRING, "
+ + " `ss_ext_tax` STRING, "
+ + " `ss_coupon_amt` STRING, "
+ + " `ss_net_paid` STRING, "
+ + " `ss_net_paid_inc_tax` STRING, "
+ + " `ss_net_profit` STRING "
+ + ")"
+ + "USING iceberg "
+ + "PARTITIONED BY (%s) "
+ + "TBLPROPERTIES ("
+ + " '%s' '%b',"
+ + " '%s' '%s',"
+ + " '%s' '%d')",
+ TABLE_NAME,
+ PARTITION_COLUMN,
+ TableProperties.MANIFEST_MERGE_ENABLED,
+ false,
+ TableProperties.DELETE_MODE,
+ RowLevelOperationMode.MERGE_ON_READ.modeName(),
+ TableProperties.FORMAT_VERSION,
+ 2);
+
+ this.table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
+ }
+
+ private void dropTable() {
+ sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
+ }
+
+ private String newWarehouseDir() {
+ return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
+ }
+
+ @FormatMethod
+ private void sql(@FormatString String query, Object... args) {
+ spark.sql(String.format(query, args));
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/UpdateProjectionBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/UpdateProjectionBenchmark.java
new file mode 100644
index 0000000000..d917eae5eb
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/UpdateProjectionBenchmark.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.apache.spark.sql.functions.current_date;
+import static org.apache.spark.sql.functions.date_add;
+import static org.apache.spark.sql.functions.expr;
+
+import com.google.errorprone.annotations.FormatMethod;
+import com.google.errorprone.annotations.FormatString;
+import java.util.UUID;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.DistributionMode;
+import org.apache.iceberg.RowLevelOperationMode;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.parser.ParseException;
+import org.apache.spark.sql.internal.SQLConf;
+import org.apache.spark.sql.types.StructType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.SingleShotTime)
+public class UpdateProjectionBenchmark {
+
+ private static final String TABLE_NAME = "test_table";
+ private static final int NUM_FILES = 5;
+ private static final int NUM_ROWS_PER_FILE = 1_000_000;
+
+ private final Configuration hadoopConf = new Configuration();
+ private SparkSession spark;
+ private long originalSnapshotId;
+
+ @Setup
+ public void setupBenchmark() throws NoSuchTableException, ParseException {
+ setupSpark();
+ initTable();
+ appendData();
+
+ Table table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
+ this.originalSnapshotId = table.currentSnapshot().snapshotId();
+ }
+
+ @TearDown
+ public void tearDownBenchmark() {
+ tearDownSpark();
+ dropTable();
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void copyOnWriteUpdate10Percent() {
+ runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.1);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void copyOnWriteUpdate30Percent() {
+ runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.3);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void copyOnWriteUpdate75Percent() {
+ runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.75);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void mergeOnRead10Percent() {
+ runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.1);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void mergeOnReadUpdate30Percent() {
+ runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.3);
+ }
+
+ @Benchmark
+ @Threads(1)
+ public void mergeOnReadUpdate75Percent() {
+ runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.75);
+ }
+
+ private void runBenchmark(RowLevelOperationMode mode, double updatePercentage) {
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')",
+ TABLE_NAME, TableProperties.UPDATE_MODE, mode.modeName());
+
+ int mod = (int) (NUM_ROWS_PER_FILE / (NUM_ROWS_PER_FILE * updatePercentage));
+
+ sql(
+ "UPDATE %s "
+ + "SET intCol = intCol + 10, dateCol = date_add(dateCol, 1) "
+ + "WHERE mod(id, %d) = 0",
+ TABLE_NAME, mod);
+
+ sql(
+ "CALL system.rollback_to_snapshot(table => '%s', snapshot_id => %dL)",
+ TABLE_NAME, originalSnapshotId);
+ }
+
+ private void setupSpark() {
+ this.spark =
+ SparkSession.builder()
+ .config("spark.ui.enabled", false)
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.type", "hadoop")
+ .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
+ .config(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key(), "false")
+ .config(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false")
+ .config(SQLConf.SHUFFLE_PARTITIONS().key(), "2")
+ .master("local")
+ .getOrCreate();
+ }
+
+ private void tearDownSpark() {
+ spark.stop();
+ }
+
+ private void initTable() {
+ sql(
+ "CREATE TABLE %s ( "
+ + " id LONG, intCol INT, floatCol FLOAT, doubleCol DOUBLE, "
+ + " decimalCol DECIMAL(20, 5), dateCol DATE, timestampCol TIMESTAMP, "
+ + " stringCol STRING)"
+ + "USING iceberg "
+ + "TBLPROPERTIES ("
+ + " '%s' '%s',"
+ + " '%s' '%d',"
+ + " '%s' '%d')",
+ TABLE_NAME,
+ TableProperties.UPDATE_DISTRIBUTION_MODE,
+ DistributionMode.NONE.modeName(),
+ TableProperties.SPLIT_OPEN_FILE_COST,
+ Integer.MAX_VALUE,
+ TableProperties.FORMAT_VERSION,
+ 2);
+
+ sql("ALTER TABLE %s WRITE ORDERED BY id", TABLE_NAME);
+ }
+
+ private void dropTable() {
+ sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
+ }
+
+ private void appendData() throws NoSuchTableException {
+ for (int fileNum = 1; fileNum <= NUM_FILES; fileNum++) {
+ Dataset<Row> inputDF =
+ spark
+ .range(NUM_ROWS_PER_FILE)
+ .withColumn("intCol", expr("CAST(id AS INT)"))
+ .withColumn("floatCol", expr("CAST(id AS FLOAT)"))
+ .withColumn("doubleCol", expr("CAST(id AS DOUBLE)"))
+ .withColumn("decimalCol", expr("CAST(id AS DECIMAL(20, 5))"))
+ .withColumn("dateCol", date_add(current_date(), fileNum))
+ .withColumn("timestampCol", expr("TO_TIMESTAMP(dateCol)"))
+ .withColumn("stringCol", expr("CAST(dateCol AS STRING)"));
+ appendAsFile(inputDF);
+ }
+ }
+
+ private void appendAsFile(Dataset<Row> df) throws NoSuchTableException {
+ // ensure the schema is precise (including nullability)
+ StructType sparkSchema = spark.table(TABLE_NAME).schema();
+ spark.createDataFrame(df.rdd(), sparkSchema).coalesce(1).writeTo(TABLE_NAME).append();
+ }
+
+ private String newWarehouseDir() {
+ return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
+ }
+
+ @FormatMethod
+ private void sql(@FormatString String query, Object... args) {
+ spark.sql(String.format(query, args));
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 b/spark/v4.0/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
new file mode 100644
index 0000000000..4c2a16d7b1
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * This file is an adaptation of Presto's and Spark's grammar files.
+ */
+
+grammar IcebergSqlExtensions;
+
+@lexer::members {
+ /**
+ * Verify whether current token is a valid decimal token (which contains dot).
+ * Returns true if the character that follows the token is not a digit or letter or underscore.
+ *
+ * For example:
+ * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'.
+ * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'.
+ * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'.
+ * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed
+ * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+'
+ * which is not a digit or letter or underscore.
+ */
+ public boolean isValidDecimal() {
+ int nextChar = _input.LA(1);
+ if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' ||
+ nextChar == '_') {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * This method will be called when we see '/*' and try to match it as a bracketed comment.
+ * If the next character is '+', it should be parsed as hint later, and we cannot match
+ * it as a bracketed comment.
+ *
+ * Returns true if the next character is '+'.
+ */
+ public boolean isHint() {
+ int nextChar = _input.LA(1);
+ if (nextChar == '+') {
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
+
+singleStatement
+ : statement EOF
+ ;
+
+statement
+ : ALTER TABLE multipartIdentifier ADD PARTITION FIELD transform (AS name=identifier)? #addPartitionField
+ | ALTER TABLE multipartIdentifier DROP PARTITION FIELD transform #dropPartitionField
+ | ALTER TABLE multipartIdentifier REPLACE PARTITION FIELD transform WITH transform (AS name=identifier)? #replacePartitionField
+ | ALTER TABLE multipartIdentifier WRITE writeSpec #setWriteDistributionAndOrdering
+ | ALTER TABLE multipartIdentifier SET IDENTIFIER_KW FIELDS fieldList #setIdentifierFields
+ | ALTER TABLE multipartIdentifier DROP IDENTIFIER_KW FIELDS fieldList #dropIdentifierFields
+ | ALTER TABLE multipartIdentifier createReplaceBranchClause #createOrReplaceBranch
+ | ALTER TABLE multipartIdentifier createReplaceTagClause #createOrReplaceTag
+ | ALTER TABLE multipartIdentifier DROP BRANCH (IF EXISTS)? identifier #dropBranch
+ | ALTER TABLE multipartIdentifier DROP TAG (IF EXISTS)? identifier #dropTag
+ ;
+
+createReplaceTagClause
+ : (CREATE OR)? REPLACE TAG identifier tagOptions
+ | CREATE TAG (IF NOT EXISTS)? identifier tagOptions
+ ;
+
+createReplaceBranchClause
+ : (CREATE OR)? REPLACE BRANCH identifier branchOptions
+ | CREATE BRANCH (IF NOT EXISTS)? identifier branchOptions
+ ;
+
+tagOptions
+ : (AS OF VERSION snapshotId)? (refRetain)?
+ ;
+
+branchOptions
+ : (AS OF VERSION snapshotId)? (refRetain)? (snapshotRetention)?
+ ;
+
+snapshotRetention
+ : WITH SNAPSHOT RETENTION minSnapshotsToKeep
+ | WITH SNAPSHOT RETENTION maxSnapshotAge
+ | WITH SNAPSHOT RETENTION minSnapshotsToKeep maxSnapshotAge
+ ;
+
+refRetain
+ : RETAIN number timeUnit
+ ;
+
+maxSnapshotAge
+ : number timeUnit
+ ;
+
+minSnapshotsToKeep
+ : number SNAPSHOTS
+ ;
+
+writeSpec
+ : (writeDistributionSpec | writeOrderingSpec)*
+ ;
+
+writeDistributionSpec
+ : DISTRIBUTED BY PARTITION
+ ;
+
+writeOrderingSpec
+ : LOCALLY? ORDERED BY order
+ | UNORDERED
+ ;
+
+singleOrder
+ : order EOF
+ ;
+
+order
+ : fields+=orderField (',' fields+=orderField)*
+ | '(' fields+=orderField (',' fields+=orderField)* ')'
+ ;
+
+orderField
+ : transform direction=(ASC | DESC)? (NULLS nullOrder=(FIRST | LAST))?
+ ;
+
+transform
+ : multipartIdentifier #identityTransform
+ | transformName=identifier
+ '(' arguments+=transformArgument (',' arguments+=transformArgument)* ')' #applyTransform
+ ;
+
+transformArgument
+ : multipartIdentifier
+ | constant
+ ;
+
+expression
+ : constant
+ | stringMap
+ | stringArray
+ ;
+
+constant
+ : number #numericLiteral
+ | booleanValue #booleanLiteral
+ | STRING+ #stringLiteral
+ | identifier STRING #typeConstructor
+ ;
+
+stringMap
+ : MAP '(' constant (',' constant)* ')'
+ ;
+
+stringArray
+ : ARRAY '(' constant (',' constant)* ')'
+ ;
+
+booleanValue
+ : TRUE | FALSE
+ ;
+
+number
+ : MINUS? EXPONENT_VALUE #exponentLiteral
+ | MINUS? DECIMAL_VALUE #decimalLiteral
+ | MINUS? INTEGER_VALUE #integerLiteral
+ | MINUS? BIGINT_LITERAL #bigIntLiteral
+ | MINUS? SMALLINT_LITERAL #smallIntLiteral
+ | MINUS? TINYINT_LITERAL #tinyIntLiteral
+ | MINUS? DOUBLE_LITERAL #doubleLiteral
+ | MINUS? FLOAT_LITERAL #floatLiteral
+ | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral
+ ;
+
+multipartIdentifier
+ : parts+=identifier ('.' parts+=identifier)*
+ ;
+
+identifier
+ : IDENTIFIER #unquotedIdentifier
+ | quotedIdentifier #quotedIdentifierAlternative
+ | nonReserved #unquotedIdentifier
+ ;
+
+quotedIdentifier
+ : BACKQUOTED_IDENTIFIER
+ ;
+
+fieldList
+ : fields+=multipartIdentifier (',' fields+=multipartIdentifier)*
+ ;
+
+nonReserved
+ : ADD | ALTER | AS | ASC | BRANCH | BY | CREATE | DAYS | DESC | DROP | EXISTS | FIELD | FIRST | HOURS | IF | LAST | NOT | NULLS | OF | OR | ORDERED | PARTITION | TABLE | WRITE
+ | DISTRIBUTED | LOCALLY | MINUTES | MONTHS | UNORDERED | REPLACE | RETAIN | VERSION | WITH | IDENTIFIER_KW | FIELDS | SET | SNAPSHOT | SNAPSHOTS
+ | TAG | TRUE | FALSE
+ | MAP
+ ;
+
+snapshotId
+ : number
+ ;
+
+numSnapshots
+ : number
+ ;
+
+timeUnit
+ : DAYS
+ | HOURS
+ | MINUTES
+ ;
+
+ADD: 'ADD';
+ALTER: 'ALTER';
+AS: 'AS';
+ASC: 'ASC';
+BRANCH: 'BRANCH';
+BY: 'BY';
+DAYS: 'DAYS';
+DESC: 'DESC';
+DISTRIBUTED: 'DISTRIBUTED';
+DROP: 'DROP';
+EXISTS: 'EXISTS';
+FIELD: 'FIELD';
+FIELDS: 'FIELDS';
+FIRST: 'FIRST';
+HOURS: 'HOURS';
+IF : 'IF';
+LAST: 'LAST';
+LOCALLY: 'LOCALLY';
+MINUTES: 'MINUTES';
+MONTHS: 'MONTHS';
+CREATE: 'CREATE';
+NOT: 'NOT';
+NULLS: 'NULLS';
+OF: 'OF';
+OR: 'OR';
+ORDERED: 'ORDERED';
+PARTITION: 'PARTITION';
+REPLACE: 'REPLACE';
+RETAIN: 'RETAIN';
+RETENTION: 'RETENTION';
+IDENTIFIER_KW: 'IDENTIFIER';
+SET: 'SET';
+SNAPSHOT: 'SNAPSHOT';
+SNAPSHOTS: 'SNAPSHOTS';
+TABLE: 'TABLE';
+TAG: 'TAG';
+UNORDERED: 'UNORDERED';
+VERSION: 'VERSION';
+WITH: 'WITH';
+WRITE: 'WRITE';
+
+TRUE: 'TRUE';
+FALSE: 'FALSE';
+
+MAP: 'MAP';
+ARRAY: 'ARRAY';
+
+PLUS: '+';
+MINUS: '-';
+
+STRING
+ : '\'' ( ~('\''|'\\') | ('\\' .) )* '\''
+ | '"' ( ~('"'|'\\') | ('\\' .) )* '"'
+ ;
+
+BIGINT_LITERAL
+ : DIGIT+ 'L'
+ ;
+
+SMALLINT_LITERAL
+ : DIGIT+ 'S'
+ ;
+
+TINYINT_LITERAL
+ : DIGIT+ 'Y'
+ ;
+
+INTEGER_VALUE
+ : DIGIT+
+ ;
+
+EXPONENT_VALUE
+ : DIGIT+ EXPONENT
+ | DECIMAL_DIGITS EXPONENT {isValidDecimal()}?
+ ;
+
+DECIMAL_VALUE
+ : DECIMAL_DIGITS {isValidDecimal()}?
+ ;
+
+FLOAT_LITERAL
+ : DIGIT+ EXPONENT? 'F'
+ | DECIMAL_DIGITS EXPONENT? 'F' {isValidDecimal()}?
+ ;
+
+DOUBLE_LITERAL
+ : DIGIT+ EXPONENT? 'D'
+ | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}?
+ ;
+
+BIGDECIMAL_LITERAL
+ : DIGIT+ EXPONENT? 'BD'
+ | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}?
+ ;
+
+IDENTIFIER
+ : (LETTER | DIGIT | '_')+
+ ;
+
+BACKQUOTED_IDENTIFIER
+ : '`' ( ~'`' | '``' )* '`'
+ ;
+
+fragment DECIMAL_DIGITS
+ : DIGIT+ '.' DIGIT*
+ | '.' DIGIT+
+ ;
+
+fragment EXPONENT
+ : 'E' [+-]? DIGIT+
+ ;
+
+fragment DIGIT
+ : [0-9]
+ ;
+
+fragment LETTER
+ : [A-Z]
+ ;
+
+SIMPLE_COMMENT
+ : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
+ ;
+
+BRACKETED_COMMENT
+ : '/*' {!isHint()}? (BRACKETED_COMMENT|.)*? '*/' -> channel(HIDDEN)
+ ;
+
+WS
+ : [ \r\n\t]+ -> channel(HIDDEN)
+ ;
+
+// Catch-all for anything we can't recognize.
+// We use this to be able to ignore and recover all the text
+// when splitting statements with DelimiterLexer
+UNRECOGNIZED
+ : .
+ ;
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
new file mode 100644
index 0000000000..c4de35010c
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions
+
+import org.apache.spark.sql.SparkSessionExtensions
+import org.apache.spark.sql.catalyst.analysis.CheckViews
+import org.apache.spark.sql.catalyst.analysis.ResolveViews
+import org.apache.spark.sql.catalyst.optimizer.ReplaceStaticInvoke
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser
+import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy
+
+class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
+
+ override def apply(extensions: SparkSessionExtensions): Unit = {
+ // parser extensions
+ extensions.injectParser { case (_, parser) => new IcebergSparkSqlExtensionsParser(parser) }
+
+ // analyzer extensions
+ extensions.injectResolutionRule { spark => ResolveViews(spark) }
+ extensions.injectCheckRule(_ => CheckViews)
+
+ // optimizer extensions
+ extensions.injectOptimizerRule { _ => ReplaceStaticInvoke }
+
+ // planner extensions
+ extensions.injectPlannerStrategy { spark => ExtendedDataSourceV2Strategy(spark) }
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
new file mode 100644
index 0000000000..5ad4b9c014
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.plans.logical.AlterViewAs
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias
+import org.apache.spark.sql.catalyst.plans.logical.View
+import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
+import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.util.SchemaUtils
+
+object CheckViews extends (LogicalPlan => Unit) {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override def apply(plan: LogicalPlan): Unit = {
+ plan foreach {
+ case CreateIcebergView(
+ resolvedIdent @ ResolvedIdentifier(_: ViewCatalog, _),
+ _,
+ query,
+ columnAliases,
+ _,
+ _,
+ _,
+ _,
+ _,
+ replace,
+ _,
+ _) =>
+ verifyColumnCount(resolvedIdent, columnAliases, query)
+ SchemaUtils.checkColumnNameDuplication(
+ query.schema.fieldNames.toIndexedSeq,
+ SQLConf.get.resolver)
+ if (replace) {
+ val viewIdent: Seq[String] =
+ resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
+ checkCyclicViewReference(viewIdent, query, Seq(viewIdent))
+ }
+
+ case AlterViewAs(ResolvedV2View(_, _), _, _) =>
+ throw new IcebergAnalysisException(
+ "ALTER VIEW <viewName> AS is not supported. Use CREATE OR REPLACE VIEW instead")
+
+ case _ => // OK
+ }
+ }
+
+ private def verifyColumnCount(
+ ident: ResolvedIdentifier,
+ columns: Seq[String],
+ query: LogicalPlan): Unit = {
+ if (columns.nonEmpty) {
+ if (columns.length > query.output.length) {
+ throw new AnalysisException(
+ errorClass = "CREATE_VIEW_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS",
+ messageParameters = Map(
+ "viewName" -> String.format("%s.%s", ident.catalog.name(), ident.identifier),
+ "viewColumns" -> columns.mkString(", "),
+ "dataColumns" -> query.output.map(c => c.name).mkString(", ")))
+ } else if (columns.length < query.output.length) {
+ throw new AnalysisException(
+ errorClass = "CREATE_VIEW_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS",
+ messageParameters = Map(
+ "viewName" -> String.format("%s.%s", ident.catalog.name(), ident.identifier),
+ "viewColumns" -> columns.mkString(", "),
+ "dataColumns" -> query.output.map(c => c.name).mkString(", ")))
+ }
+ }
+ }
+
+ private def checkCyclicViewReference(
+ viewIdent: Seq[String],
+ plan: LogicalPlan,
+ cyclePath: Seq[Seq[String]]): Unit = {
+ plan match {
+ case sub @ SubqueryAlias(_, Project(_, _)) =>
+ val currentViewIdent: Seq[String] = sub.identifier.qualifier :+ sub.identifier.name
+ checkIfRecursiveView(viewIdent, currentViewIdent, cyclePath, sub.children)
+ case v1View: View =>
+ val currentViewIdent: Seq[String] = v1View.desc.identifier.nameParts
+ checkIfRecursiveView(viewIdent, currentViewIdent, cyclePath, v1View.children)
+ case _ =>
+ plan.children.foreach(child => checkCyclicViewReference(viewIdent, child, cyclePath))
+ }
+
+ plan.expressions.flatMap(_.flatMap {
+ case e: SubqueryExpression =>
+ checkCyclicViewReference(viewIdent, e.plan, cyclePath)
+ None
+ case _ => None
+ })
+ }
+
+ private def checkIfRecursiveView(
+ viewIdent: Seq[String],
+ currentViewIdent: Seq[String],
+ cyclePath: Seq[Seq[String]],
+ children: Seq[LogicalPlan]): Unit = {
+ val newCyclePath = cyclePath :+ currentViewIdent
+ if (currentViewIdent == viewIdent) {
+ throw new IcebergAnalysisException(
+ String.format(
+ "Recursive cycle in view detected: %s (cycle: %s)",
+ viewIdent.asIdentifier,
+ newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
+ } else {
+ children.foreach { c =>
+ checkCyclicViewReference(viewIdent, c, newCyclePath)
+ }
+ }
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
new file mode 100644
index 0000000000..ff7d20241b
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.analysis.ViewUtil.IcebergViewHelper
+import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.expressions.UpCast
+import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.Project
+import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias
+import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
+import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+import org.apache.spark.sql.catalyst.trees.Origin
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.connector.catalog.LookupCatalog
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.types.MetadataBuilder
+
+case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case u @ UnresolvedRelation(nameParts, _, _)
+ if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
+ u
+
+ case u @ UnresolvedRelation(parts @ CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
+ .map(createViewRelation(parts, _))
+ .getOrElse(u)
+
+ case u @ UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
+ .map(_ => ResolvedV2View(catalog.asViewCatalog, ident))
+ .getOrElse(u)
+
+ case c @ CreateIcebergView(
+ ResolvedIdentifier(_, _),
+ _,
+ query,
+ columnAliases,
+ columnComments,
+ _,
+ _,
+ _,
+ _,
+ _,
+ _,
+ _) if query.resolved && !c.rewritten =>
+ val aliased = aliasColumns(query, columnAliases, columnComments)
+ c.copy(
+ query = aliased,
+ queryColumnNames = query.schema.fieldNames.toIndexedSeq,
+ rewritten = true)
+ }
+
+ private def aliasColumns(
+ plan: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]]): LogicalPlan = {
+ if (columnAliases.isEmpty || columnAliases.length != plan.output.length) {
+ plan
+ } else {
+ val projectList = plan.output.zipWithIndex.map { case (attr, pos) =>
+ if (columnComments.apply(pos).isDefined) {
+ val meta =
+ new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
+ Alias(attr, columnAliases.apply(pos))(explicitMetadata = Some(meta))
+ } else {
+ Alias(attr, columnAliases.apply(pos))()
+ }
+ }
+ Project(projectList, plan)
+ }
+ }
+
+ private def createViewRelation(nameParts: Seq[String], view: View): LogicalPlan = {
+ val parsed = parseViewText(nameParts.quoted, view.query)
+
+ // Apply any necessary rewrites to preserve correct resolution
+ val viewCatalogAndNamespace: Seq[String] = view.currentCatalog +: view.currentNamespace.toSeq
+ val rewritten = rewriteIdentifiers(parsed, viewCatalogAndNamespace);
+
+ // Apply the field aliases and column comments
+ // This logic differs from how Spark handles views in SessionCatalog.fromCatalogTable.
+ // This is more strict because it doesn't allow resolution by field name.
+ val aliases = view.schema.fields.zipWithIndex.map { case (expected, pos) =>
+ val attr = GetColumnByOrdinal(pos, expected.dataType)
+ Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata =
+ Some(expected.metadata))
+ }.toIndexedSeq
+
+ SubqueryAlias(nameParts, Project(aliases, rewritten))
+ }
+
+ private def parseViewText(name: String, viewText: String): LogicalPlan = {
+ val origin = Origin(objectType = Some("VIEW"), objectName = Some(name))
+
+ try {
+ CurrentOrigin.withOrigin(origin) {
+ spark.sessionState.sqlParser.parseQuery(viewText)
+ }
+ } catch {
+ case _: ParseException =>
+ throw QueryCompilationErrors.invalidViewNameError(name)
+ }
+ }
+
+ private def rewriteIdentifiers(
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = {
+ // Substitute CTEs and Unresolved Ordinals within the view, then rewrite unresolved functions and relations
+ qualifyTableIdentifiers(
+ qualifyFunctionIdentifiers(
+ SubstituteUnresolvedOrdinals.apply(CTESubstitution.apply(plan)),
+ catalogAndNamespace),
+ catalogAndNamespace)
+ }
+
+ private def qualifyFunctionIdentifiers(
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
+ case u @ UnresolvedFunction(Seq(name), _, _, _, _, _, _) =>
+ if (!isBuiltinFunction(name)) {
+ u.copy(nameParts = catalogAndNamespace :+ name)
+ } else {
+ u
+ }
+ case u @ UnresolvedFunction(parts, _, _, _, _, _, _) if !isCatalog(parts.head) =>
+ u.copy(nameParts = catalogAndNamespace.head +: parts)
+ }
+
+ /**
+ * Qualify table identifiers with default catalog and namespace if necessary.
+ */
+ private def qualifyTableIdentifiers(
+ child: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan =
+ child transform {
+ case u @ UnresolvedRelation(Seq(table), _, _) =>
+ u.copy(multipartIdentifier = catalogAndNamespace :+ table)
+ case u @ UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
+ u.copy(multipartIdentifier = catalogAndNamespace.head +: parts)
+ case other =>
+ other.transformExpressions { case subquery: SubqueryExpression =>
+ subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
+ }
+ }
+
+ private def isCatalog(name: String): Boolean = {
+ catalogManager.isCatalogRegistered(name)
+ }
+
+ private def isBuiltinFunction(name: String): Boolean = {
+ catalogManager.v1SessionCatalog.isBuiltinFunction(FunctionIdentifier(name))
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
new file mode 100644
index 0000000000..ac0f75c422
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.analysis.ViewUtil.IcebergViewHelper
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.plans.logical.CreateView
+import org.apache.spark.sql.catalyst.plans.logical.DropView
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.ShowViews
+import org.apache.spark.sql.catalyst.plans.logical.View
+import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
+import org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView
+import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
+import org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.UNRESOLVED_FUNCTION
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.connector.catalog.LookupCatalog
+import scala.collection.mutable
+
+/**
+ * ResolveSessionCatalog exits early for some v2 View commands,
+ * thus they are pre-substituted here and then handled in ResolveViews
+ */
+case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+ case DropView(ResolvedIdent(resolved), ifExists) =>
+ DropIcebergView(resolved, ifExists)
+
+ case CreateView(
+ ResolvedIdent(resolved),
+ userSpecifiedColumns,
+ comment,
+ _,
+ properties,
+ Some(queryText),
+ query,
+ allowExisting,
+ replace,
+ _) =>
+ val q = CTESubstitution.apply(query)
+ verifyTemporaryObjectsDontExist(resolved, q)
+ CreateIcebergView(
+ child = resolved,
+ queryText = queryText,
+ query = q,
+ columnAliases = userSpecifiedColumns.map(_._1),
+ columnComments = userSpecifiedColumns.map(_._2.orElse(Option.empty)),
+ comment = comment,
+ properties = properties,
+ allowExisting = allowExisting,
+ replace = replace)
+
+ case view @ ShowViews(CurrentNamespace, pattern, output) =>
+ if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) {
+ ShowIcebergViews(
+ ResolvedNamespace(
+ catalogManager.currentCatalog,
+ catalogManager.currentNamespace.toIndexedSeq),
+ pattern,
+ output)
+ } else {
+ view
+ }
+
+ case ShowViews(UnresolvedNamespace(CatalogAndNamespace(catalog, ns), _), pattern, output)
+ if ViewUtil.isViewCatalog(catalog) =>
+ ShowIcebergViews(ResolvedNamespace(catalog, ns), pattern, output)
+
+ // needs to be done here instead of in ResolveViews, so that a V2 view can be resolved before the Analyzer
+ // tries to resolve it, which would result in an error, saying that V2 views aren't supported
+ case u @ UnresolvedView(ResolvedView(resolved), _, _, _) =>
+ ViewUtil
+ .loadView(resolved.catalog, resolved.identifier)
+ .map(_ => ResolvedV2View(resolved.catalog.asViewCatalog, resolved.identifier))
+ .getOrElse(u)
+ }
+
+ private def isTempView(nameParts: Seq[String]): Boolean = {
+ catalogManager.v1SessionCatalog.isTempView(nameParts)
+ }
+
+ private def isTempFunction(nameParts: Seq[String]): Boolean = {
+ if (nameParts.size > 1) {
+ return false
+ }
+ catalogManager.v1SessionCatalog.isTemporaryFunction(nameParts.asFunctionIdentifier)
+ }
+
+ private object ResolvedIdent {
+ def unapply(unresolved: UnresolvedIdentifier): Option[ResolvedIdentifier] = unresolved match {
+ case UnresolvedIdentifier(nameParts, true) if isTempView(nameParts) =>
+ None
+
+ case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _)
+ if ViewUtil.isViewCatalog(catalog) =>
+ Some(ResolvedIdentifier(catalog, ident))
+
+ case _ =>
+ None
+ }
+ }
+
+ /**
+ * Permanent views are not allowed to reference temp objects
+ */
+ private def verifyTemporaryObjectsDontExist(
+ identifier: ResolvedIdentifier,
+ child: LogicalPlan): Unit = {
+ val tempViews = collectTemporaryViews(child)
+ if (tempViews.nonEmpty) {
+ throw invalidRefToTempObject(
+ identifier,
+ tempViews.map(v => v.quoted).mkString("[", ", ", "]"),
+ "view")
+ }
+
+ val tempFunctions = collectTemporaryFunctions(child)
+ if (tempFunctions.nonEmpty) {
+ throw invalidRefToTempObject(identifier, tempFunctions.mkString("[", ", ", "]"), "function")
+ }
+ }
+
+ private def invalidRefToTempObject(
+ ident: ResolvedIdentifier,
+ tempObjectNames: String,
+ tempObjectType: String) = {
+ new IcebergAnalysisException(
+ String.format(
+ "Cannot create view %s.%s that references temporary %s: %s",
+ ident.catalog.name(),
+ ident.identifier,
+ tempObjectType,
+ tempObjectNames))
+ }
+
+ /**
+ * Collect all temporary views and return the identifiers separately
+ */
+ private def collectTemporaryViews(child: LogicalPlan): Seq[Seq[String]] = {
+ def collectTempViews(child: LogicalPlan): Seq[Seq[String]] = {
+ child.flatMap {
+ case unresolved: UnresolvedRelation if isTempView(unresolved.multipartIdentifier) =>
+ Seq(unresolved.multipartIdentifier)
+ case view: View if view.isTempView => Seq(view.desc.identifier.nameParts)
+ case plan =>
+ plan.expressions.flatMap(_.flatMap {
+ case e: SubqueryExpression => collectTempViews(e.plan)
+ case _ => Seq.empty
+ })
+ }.distinct
+ }
+
+ collectTempViews(child)
+ }
+
+ private object ResolvedView {
+ def unapply(identifier: Seq[String]): Option[ResolvedV2View] = identifier match {
+ case nameParts if isTempView(nameParts) =>
+ None
+
+ case CatalogAndIdentifier(catalog, ident) if ViewUtil.isViewCatalog(catalog) =>
+ ViewUtil
+ .loadView(catalog, ident)
+ .flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
+
+ case _ =>
+ None
+ }
+ }
+
+ /**
+ * Collect the names of all temporary functions.
+ */
+ private def collectTemporaryFunctions(child: LogicalPlan): Seq[String] = {
+ val tempFunctions = new mutable.HashSet[String]()
+ child.resolveExpressionsWithPruning(_.containsAnyPattern(UNRESOLVED_FUNCTION)) {
+ case f @ UnresolvedFunction(nameParts, _, _, _, _, _, _) if isTempFunction(nameParts) =>
+ tempFunctions += nameParts.head
+ f
+ case e: SubqueryExpression =>
+ tempFunctions ++= collectTemporaryFunctions(e.plan)
+ e
+ }
+ tempFunctions.toSeq
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
new file mode 100644
index 0000000000..a7188837c5
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.connector.catalog.CatalogPlugin
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.errors.QueryCompilationErrors
+
+object ViewUtil {
+ def loadView(catalog: CatalogPlugin, ident: Identifier): Option[View] = catalog match {
+ case viewCatalog: ViewCatalog =>
+ try {
+ Option(viewCatalog.loadView(ident))
+ } catch {
+ case _: NoSuchViewException => None
+ }
+ case _ => None
+ }
+
+ def isViewCatalog(catalog: CatalogPlugin): Boolean = {
+ catalog.isInstanceOf[ViewCatalog]
+ }
+
+ implicit class IcebergViewHelper(plugin: CatalogPlugin) {
+ def asViewCatalog: ViewCatalog = plugin match {
+ case viewCatalog: ViewCatalog =>
+ viewCatalog
+ case _ =>
+ throw QueryCompilationErrors.missingCatalogAbilityError(plugin, "views")
+ }
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
new file mode 100644
index 0000000000..bdec4aae88
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.iceberg.spark.functions.SparkFunctions
+import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression
+import org.apache.spark.sql.catalyst.expressions.BinaryComparison
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.In
+import org.apache.spark.sql.catalyst.expressions.InSet
+import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
+import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.plans.logical.Join
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.BINARY_COMPARISON
+import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND
+import org.apache.spark.sql.catalyst.trees.TreePattern.FILTER
+import org.apache.spark.sql.catalyst.trees.TreePattern.IN
+import org.apache.spark.sql.catalyst.trees.TreePattern.INSET
+import org.apache.spark.sql.catalyst.trees.TreePattern.JOIN
+import org.apache.spark.sql.connector.catalog.functions.ScalarFunction
+import org.apache.spark.sql.types.StructField
+import org.apache.spark.sql.types.StructType
+
+/**
+ * Spark analyzes the Iceberg system function to {@link StaticInvoke} which could not be pushed
+ * down to datasource. This rule will replace {@link StaticInvoke} to
+ * {@link ApplyFunctionExpression} for Iceberg system function in a filter condition.
+ */
+object ReplaceStaticInvoke extends Rule[LogicalPlan] {
+
+ override def apply(plan: LogicalPlan): LogicalPlan =
+ plan.transformWithPruning(_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
+ case replace @ ReplaceData(_, cond, _, _, _, _, _) =>
+ replaceStaticInvoke(replace, cond, newCond => replace.copy(condition = newCond))
+
+ case join @ Join(_, _, _, Some(cond), _) =>
+ replaceStaticInvoke(join, cond, newCond => join.copy(condition = Some(newCond)))
+
+ case filter @ Filter(cond, _) =>
+ replaceStaticInvoke(filter, cond, newCond => filter.copy(condition = newCond))
+ }
+
+ private def replaceStaticInvoke[T <: LogicalPlan](
+ node: T,
+ condition: Expression,
+ copy: Expression => T): T = {
+ val newCondition = replaceStaticInvoke(condition)
+ if (newCondition fastEquals condition) node else copy(newCondition)
+ }
+
+ private def replaceStaticInvoke(condition: Expression): Expression = {
+ condition.transformWithPruning(_.containsAnyPattern(BINARY_COMPARISON, IN, INSET)) {
+ case in @ In(value: StaticInvoke, _) if canReplace(value) =>
+ in.copy(value = replaceStaticInvoke(value))
+
+ case in @ InSet(value: StaticInvoke, _) if canReplace(value) =>
+ in.copy(child = replaceStaticInvoke(value))
+
+ case c @ BinaryComparison(left: StaticInvoke, right) if canReplace(left) && right.foldable =>
+ c.withNewChildren(Seq(replaceStaticInvoke(left), right))
+
+ case c @ BinaryComparison(left, right: StaticInvoke) if canReplace(right) && left.foldable =>
+ c.withNewChildren(Seq(left, replaceStaticInvoke(right)))
+ }
+ }
+
+ private def replaceStaticInvoke(invoke: StaticInvoke): Expression = {
+ // Adaptive from `resolveV2Function` in org.apache.spark.sql.catalyst.analysis.ResolveFunctions
+ val unbound = SparkFunctions.loadFunctionByClass(invoke.staticObject)
+ if (unbound == null) {
+ return invoke
+ }
+
+ val inputType = StructType(invoke.arguments.zipWithIndex.map { case (exp, pos) =>
+ StructField(s"_$pos", exp.dataType, exp.nullable)
+ })
+
+ val bound =
+ try {
+ unbound.bind(inputType)
+ } catch {
+ case _: Exception =>
+ return invoke
+ }
+
+ if (bound.inputTypes().length != invoke.arguments.length) {
+ return invoke
+ }
+
+ bound match {
+ case scalarFunc: ScalarFunction[_] =>
+ ApplyFunctionExpression(scalarFunc, invoke.arguments)
+ case _ => invoke
+ }
+ }
+
+ @inline
+ private def canReplace(invoke: StaticInvoke): Boolean = {
+ invoke.functionName == ScalarFunction.MAGIC_METHOD_NAME && !invoke.foldable
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
new file mode 100644
index 0000000000..25e056ee2d
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.parser.extensions
+
+import java.util.Locale
+import org.antlr.v4.runtime._
+import org.antlr.v4.runtime.atn.PredictionMode
+import org.antlr.v4.runtime.misc.Interval
+import org.antlr.v4.runtime.misc.ParseCancellationException
+import org.antlr.v4.runtime.tree.TerminalNodeImpl
+import org.apache.iceberg.common.DynConstructors
+import org.apache.iceberg.spark.ExtendedParser
+import org.apache.iceberg.spark.ExtendedParser.RawOrderField
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.RewriteViewCommands
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.trees.Origin
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.VariableSubstitution
+import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types.StructType
+import scala.jdk.CollectionConverters._
+
+class IcebergSparkSqlExtensionsParser(delegate: ParserInterface)
+ extends ParserInterface
+ with ExtendedParser {
+
+ import IcebergSparkSqlExtensionsParser._
+
+ private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get)
+ private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate)
+
+ /**
+ * Parse a string to a DataType.
+ */
+ override def parseDataType(sqlText: String): DataType = {
+ delegate.parseDataType(sqlText)
+ }
+
+ /**
+ * Parse a string to a raw DataType without CHAR/VARCHAR replacement.
+ */
+ def parseRawDataType(sqlText: String): DataType = throw new UnsupportedOperationException()
+
+ /**
+ * Parse a string to an Expression.
+ */
+ override def parseExpression(sqlText: String): Expression = {
+ delegate.parseExpression(sqlText)
+ }
+
+ /**
+ * Parse a string to a TableIdentifier.
+ */
+ override def parseTableIdentifier(sqlText: String): TableIdentifier = {
+ delegate.parseTableIdentifier(sqlText)
+ }
+
+ /**
+ * Parse a string to a FunctionIdentifier.
+ */
+ override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = {
+ delegate.parseFunctionIdentifier(sqlText)
+ }
+
+ /**
+ * Parse a string to a multi-part identifier.
+ */
+ override def parseMultipartIdentifier(sqlText: String): Seq[String] = {
+ delegate.parseMultipartIdentifier(sqlText)
+ }
+
+ /**
+ * Creates StructType for a given SQL string, which is a comma separated list of field
+ * definitions which will preserve the correct Hive metadata.
+ */
+ override def parseTableSchema(sqlText: String): StructType = {
+ delegate.parseTableSchema(sqlText)
+ }
+
+ override def parseSortOrder(sqlText: String): java.util.List[RawOrderField] = {
+ val fields = parse(sqlText) { parser => astBuilder.visitSingleOrder(parser.singleOrder()) }
+ fields.map { field =>
+ val (term, direction, order) = field
+ new RawOrderField(term, direction, order)
+ }.asJava
+ }
+
+ override def parseRoutineParam(sqlText: String): StructType =
+ throw new UnsupportedOperationException()
+
+ /**
+ * Parse a string to a LogicalPlan.
+ */
+ override def parsePlan(sqlText: String): LogicalPlan = {
+ val sqlTextAfterSubstitution = substitutor.substitute(sqlText)
+ if (isIcebergCommand(sqlTextAfterSubstitution)) {
+ parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }
+ .asInstanceOf[LogicalPlan]
+ } else {
+ RewriteViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText))
+ }
+ }
+
+ private def isIcebergCommand(sqlText: String): Boolean = {
+ val normalized = sqlText
+ .toLowerCase(Locale.ROOT)
+ .trim()
+ // Strip simple SQL comments that terminate a line, e.g. comments starting with `--` .
+ .replaceAll("--.*?\\n", " ")
+ // Strip newlines.
+ .replaceAll("\\s+", " ")
+ // Strip comments of the form /* ... */. This must come after stripping newlines so that
+ // comments that span multiple lines are caught.
+ .replaceAll("/\\*.*?\\*/", " ")
+ // Strip backtick then `system`.`ancestors_of` changes to system.ancestors_of
+ .replaceAll("`", "")
+ .trim()
+
+ normalized.startsWith("alter table") && (normalized.contains("add partition field") ||
+ normalized.contains("drop partition field") ||
+ normalized.contains("replace partition field") ||
+ normalized.contains("write ordered by") ||
+ normalized.contains("write locally ordered by") ||
+ normalized.contains("write distributed by") ||
+ normalized.contains("write unordered") ||
+ normalized.contains("set identifier fields") ||
+ normalized.contains("drop identifier fields") ||
+ isSnapshotRefDdl(normalized))
+ }
+
+ private def isSnapshotRefDdl(normalized: String): Boolean = {
+ normalized.contains("create branch") ||
+ normalized.contains("replace branch") ||
+ normalized.contains("create tag") ||
+ normalized.contains("replace tag") ||
+ normalized.contains("drop branch") ||
+ normalized.contains("drop tag")
+ }
+
+ protected def parse[T](command: String)(toResult: IcebergSqlExtensionsParser => T): T = {
+ val lexer = new IcebergSqlExtensionsLexer(
+ new UpperCaseCharStream(CharStreams.fromString(command)))
+ lexer.removeErrorListeners()
+ lexer.addErrorListener(IcebergParseErrorListener)
+
+ val tokenStream = new CommonTokenStream(lexer)
+ val parser = new IcebergSqlExtensionsParser(tokenStream)
+ parser.addParseListener(IcebergSqlExtensionsPostProcessor)
+ parser.removeErrorListeners()
+ parser.addErrorListener(IcebergParseErrorListener)
+
+ // https://github.com/antlr/antlr4/issues/192#issuecomment-15238595
+ // Save a great deal of time on correct inputs by using a two-stage parsing strategy.
+ try {
+ try {
+ // first, try parsing with potentially faster SLL mode and BailErrorStrategy
+ parser.setErrorHandler(new BailErrorStrategy)
+ parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
+ toResult(parser)
+ } catch {
+ case _: ParseCancellationException =>
+ // if we fail, parse with LL mode with DefaultErrorStrategy
+ tokenStream.seek(0) // rewind input stream
+ parser.reset()
+
+ // Try Again.
+ parser.setErrorHandler(new DefaultErrorStrategy)
+ parser.getInterpreter.setPredictionMode(PredictionMode.LL)
+ toResult(parser)
+ }
+ } catch {
+ case e: IcebergParseException if e.command.isDefined =>
+ throw e
+ case e: IcebergParseException =>
+ throw e.withCommand(command)
+ case e: AnalysisException =>
+ val position = Origin(e.line, e.startPosition)
+ throw new IcebergParseException(Option(command), e.message, position, position)
+ }
+ }
+
+ override def parseQuery(sqlText: String): LogicalPlan = {
+ parsePlan(sqlText)
+ }
+}
+
+object IcebergSparkSqlExtensionsParser {
+ private val substitutorCtor: DynConstructors.Ctor[VariableSubstitution] =
+ DynConstructors
+ .builder()
+ .impl(classOf[VariableSubstitution])
+ .impl(classOf[VariableSubstitution], classOf[SQLConf])
+ .build()
+}
+
+/* Copied from Apache Spark's to avoid dependency on Spark Internals */
+class UpperCaseCharStream(wrapped: CodePointCharStream) extends CharStream {
+ override def consume(): Unit = wrapped.consume
+ override def getSourceName(): String = wrapped.getSourceName
+ override def index(): Int = wrapped.index
+ override def mark(): Int = wrapped.mark
+ override def release(marker: Int): Unit = wrapped.release(marker)
+ override def seek(where: Int): Unit = wrapped.seek(where)
+ override def size(): Int = wrapped.size
+
+ override def getText(interval: Interval): String = wrapped.getText(interval)
+
+ // scalastyle:off
+ override def LA(i: Int): Int = {
+ val la = wrapped.LA(i)
+ if (la == 0 || la == IntStream.EOF) la
+ else Character.toUpperCase(la)
+ }
+ // scalastyle:on
+}
+
+/**
+ * The post-processor validates & cleans-up the parse tree during the parse process.
+ */
+case object IcebergSqlExtensionsPostProcessor extends IcebergSqlExtensionsBaseListener {
+
+ /** Remove the back ticks from an Identifier. */
+ override def exitQuotedIdentifier(ctx: QuotedIdentifierContext): Unit = {
+ replaceTokenByIdentifier(ctx, 1) { token =>
+ // Remove the double back ticks in the string.
+ token.setText(token.getText.replace("``", "`"))
+ token
+ }
+ }
+
+ /** Treat non-reserved keywords as Identifiers. */
+ override def exitNonReserved(ctx: NonReservedContext): Unit = {
+ replaceTokenByIdentifier(ctx, 0)(identity)
+ }
+
+ private def replaceTokenByIdentifier(ctx: ParserRuleContext, stripMargins: Int)(
+ f: CommonToken => CommonToken = identity): Unit = {
+ val parent = ctx.getParent
+ parent.removeLastChild()
+ val token = ctx.getChild(0).getPayload.asInstanceOf[Token]
+ val newToken = new CommonToken(
+ new org.antlr.v4.runtime.misc.Pair(token.getTokenSource, token.getInputStream),
+ IcebergSqlExtensionsParser.IDENTIFIER,
+ token.getChannel,
+ token.getStartIndex + stripMargins,
+ token.getStopIndex - stripMargins)
+ parent.addChild(new TerminalNodeImpl(f(newToken)))
+ }
+}
+
+/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
+case object IcebergParseErrorListener extends BaseErrorListener {
+ override def syntaxError(
+ recognizer: Recognizer[_, _],
+ offendingSymbol: scala.Any,
+ line: Int,
+ charPositionInLine: Int,
+ msg: String,
+ e: RecognitionException): Unit = {
+ val (start, stop) = offendingSymbol match {
+ case token: CommonToken =>
+ val start = Origin(Some(line), Some(token.getCharPositionInLine))
+ val length = token.getStopIndex - token.getStartIndex + 1
+ val stop = Origin(Some(line), Some(token.getCharPositionInLine + length))
+ (start, stop)
+ case _ =>
+ val start = Origin(Some(line), Some(charPositionInLine))
+ (start, start)
+ }
+ throw new IcebergParseException(None, msg, start, stop)
+ }
+}
+
+/**
+ * Copied from Apache Spark
+ * A [[ParseException]] is an [[AnalysisException]] that is thrown during the parse process. It
+ * contains fields and an extended error message that make reporting and diagnosing errors easier.
+ */
+class IcebergParseException(
+ val command: Option[String],
+ message: String,
+ val start: Origin,
+ val stop: Origin)
+ extends AnalysisException(message, start.line, start.startPosition) {
+
+ def this(message: String, ctx: ParserRuleContext) = {
+ this(
+ Option(IcebergParserUtils.command(ctx)),
+ message,
+ IcebergParserUtils.position(ctx.getStart),
+ IcebergParserUtils.position(ctx.getStop))
+ }
+
+ override def getMessage: String = {
+ val builder = new StringBuilder
+ builder ++= "\n" ++= message
+ start match {
+ case Origin(Some(l), Some(p), Some(_), Some(_), Some(_), Some(_), Some(_), _, _) =>
+ builder ++= s"(line $l, pos $p)\n"
+ command.foreach { cmd =>
+ val (above, below) = cmd.split("\n").splitAt(l)
+ builder ++= "\n== SQL ==\n"
+ above.foreach(builder ++= _ += '\n')
+ builder ++= (0 until p).map(_ => "-").mkString("") ++= "^^^\n"
+ below.foreach(builder ++= _ += '\n')
+ }
+ case _ =>
+ command.foreach { cmd =>
+ builder ++= "\n== SQL ==\n" ++= cmd
+ }
+ }
+ builder.toString
+ }
+
+ def withCommand(cmd: String): IcebergParseException = {
+ new IcebergParseException(Option(cmd), message, start, stop)
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
new file mode 100644
index 0000000000..724101cfe1
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.parser.extensions
+
+import java.util.Locale
+import java.util.concurrent.TimeUnit
+import org.antlr.v4.runtime._
+import org.antlr.v4.runtime.misc.Interval
+import org.antlr.v4.runtime.tree.ParseTree
+import org.antlr.v4.runtime.tree.TerminalNode
+import org.apache.iceberg.DistributionMode
+import org.apache.iceberg.NullOrder
+import org.apache.iceberg.SortDirection
+import org.apache.iceberg.expressions.Term
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.spark.sql.catalyst.analysis.IcebergAnalysisException
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergParserUtils.withOrigin
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser._
+import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.BranchOptions
+import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch
+import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag
+import org.apache.spark.sql.catalyst.plans.logical.DropBranch
+import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.DropTag
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
+import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
+import org.apache.spark.sql.catalyst.plans.logical.TagOptions
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+import org.apache.spark.sql.catalyst.trees.Origin
+import org.apache.spark.sql.connector.expressions
+import org.apache.spark.sql.connector.expressions.ApplyTransform
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.IdentityTransform
+import org.apache.spark.sql.connector.expressions.LiteralValue
+import org.apache.spark.sql.connector.expressions.Transform
+import scala.jdk.CollectionConverters._
+
+class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface)
+ extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
+
+ private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] =
+ list.asScala
+ private def toSeq[T](list: java.util.List[T]): Seq[T] = toBuffer(list).toSeq
+
+ /**
+ * Create an ADD PARTITION FIELD logical command.
+ */
+ override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField =
+ withOrigin(ctx) {
+ AddPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform),
+ Option(ctx.name).map(_.getText))
+ }
+
+ /**
+ * Create a DROP PARTITION FIELD logical command.
+ */
+ override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField =
+ withOrigin(ctx) {
+ DropPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform))
+ }
+
+ /**
+ * Create a CREATE OR REPLACE BRANCH logical command.
+ */
+ override def visitCreateOrReplaceBranch(
+ ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
+ val createOrReplaceBranchClause = ctx.createReplaceBranchClause()
+
+ val branchName = createOrReplaceBranchClause.identifier()
+ val branchOptionsContext = Option(createOrReplaceBranchClause.branchOptions())
+ val snapshotId = branchOptionsContext
+ .flatMap(branchOptions => Option(branchOptions.snapshotId()))
+ .map(_.getText.toLong)
+ val snapshotRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
+ val minSnapshotsToKeep = snapshotRetention
+ .flatMap(retention => Option(retention.minSnapshotsToKeep()))
+ .map(minSnapshots => minSnapshots.number().getText.toLong)
+ val maxSnapshotAgeMs = snapshotRetention
+ .flatMap(retention => Option(retention.maxSnapshotAge()))
+ .map(retention =>
+ TimeUnit
+ .valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retention.number().getText.toLong))
+ val branchRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
+ val branchRefAgeMs = branchRetention.map(retain =>
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
+ val create = createOrReplaceBranchClause.CREATE() != null
+ val replace = ctx.createReplaceBranchClause().REPLACE() != null
+ val ifNotExists = createOrReplaceBranchClause.EXISTS() != null
+
+ val branchOptions =
+ BranchOptions(snapshotId, minSnapshotsToKeep, maxSnapshotAgeMs, branchRefAgeMs)
+
+ CreateOrReplaceBranch(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ branchName.getText,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists)
+ }
+
+ /**
+ * Create an CREATE OR REPLACE TAG logical command.
+ */
+ override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag =
+ withOrigin(ctx) {
+ val createTagClause = ctx.createReplaceTagClause()
+
+ val tagName = createTagClause.identifier().getText
+
+ val tagOptionsContext = Option(createTagClause.tagOptions())
+ val snapshotId = tagOptionsContext
+ .flatMap(tagOptions => Option(tagOptions.snapshotId()))
+ .map(_.getText.toLong)
+ val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
+ val tagRefAgeMs = tagRetain.map(retain =>
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
+ val tagOptions = TagOptions(snapshotId, tagRefAgeMs)
+
+ val create = createTagClause.CREATE() != null
+ val replace = createTagClause.REPLACE() != null
+ val ifNotExists = createTagClause.EXISTS() != null
+
+ CreateOrReplaceTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ tagName,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists)
+ }
+
+ /**
+ * Create an DROP BRANCH logical command.
+ */
+ override def visitDropBranch(ctx: DropBranchContext): DropBranch = withOrigin(ctx) {
+ DropBranch(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
+ }
+
+ /**
+ * Create an DROP TAG logical command.
+ */
+ override def visitDropTag(ctx: DropTagContext): DropTag = withOrigin(ctx) {
+ DropTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
+ }
+
+ /**
+ * Create an REPLACE PARTITION FIELD logical command.
+ */
+ override def visitReplacePartitionField(
+ ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
+ ReplacePartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform(0)),
+ typedVisit[Transform](ctx.transform(1)),
+ Option(ctx.name).map(_.getText))
+ }
+
+ /**
+ * Create an SET IDENTIFIER FIELDS logical command.
+ */
+ override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields =
+ withOrigin(ctx) {
+ SetIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
+
+ /**
+ * Create an DROP IDENTIFIER FIELDS logical command.
+ */
+ override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields =
+ withOrigin(ctx) {
+ DropIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
+
+ /**
+ * Create a [[SetWriteDistributionAndOrdering]] for changing the write distribution and ordering.
+ */
+ override def visitSetWriteDistributionAndOrdering(
+ ctx: SetWriteDistributionAndOrderingContext): SetWriteDistributionAndOrdering = {
+
+ val tableName = typedVisit[Seq[String]](ctx.multipartIdentifier)
+
+ val (distributionSpec, orderingSpec) = toDistributionAndOrderingSpec(ctx.writeSpec)
+
+ if (distributionSpec == null && orderingSpec == null) {
+ throw new IcebergAnalysisException(
+ "ALTER TABLE has no changes: missing both distribution and ordering clauses")
+ }
+
+ val distributionMode = if (distributionSpec != null) {
+ Some(DistributionMode.HASH)
+ } else if (orderingSpec.UNORDERED != null) {
+ Some(DistributionMode.NONE)
+ } else if (orderingSpec.LOCALLY() != null) {
+ None
+ } else {
+ Some(DistributionMode.RANGE)
+ }
+
+ val ordering = if (orderingSpec != null && orderingSpec.order != null) {
+ toSeq(orderingSpec.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
+ } else {
+ Seq.empty
+ }
+
+ SetWriteDistributionAndOrdering(tableName, distributionMode, ordering)
+ }
+
+ private def toDistributionAndOrderingSpec(
+ writeSpec: WriteSpecContext): (WriteDistributionSpecContext, WriteOrderingSpecContext) = {
+
+ if (writeSpec.writeDistributionSpec.size > 1) {
+ throw new IcebergAnalysisException("ALTER TABLE contains multiple distribution clauses")
+ }
+
+ if (writeSpec.writeOrderingSpec.size > 1) {
+ throw new IcebergAnalysisException("ALTER TABLE contains multiple ordering clauses")
+ }
+
+ val distributionSpec = toBuffer(writeSpec.writeDistributionSpec).headOption.orNull
+ val orderingSpec = toBuffer(writeSpec.writeOrderingSpec).headOption.orNull
+
+ (distributionSpec, orderingSpec)
+ }
+
+ /**
+ * Create an order field.
+ */
+ override def visitOrderField(ctx: OrderFieldContext): (Term, SortDirection, NullOrder) = {
+ val term = Spark3Util.toIcebergTerm(typedVisit[Transform](ctx.transform))
+ val direction = Option(ctx.ASC)
+ .map(_ => SortDirection.ASC)
+ .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
+ .getOrElse(SortDirection.ASC)
+ val nullOrder = Option(ctx.FIRST)
+ .map(_ => NullOrder.NULLS_FIRST)
+ .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
+ .getOrElse(
+ if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
+ (term, direction, nullOrder)
+ }
+
+ /**
+ * Create an IdentityTransform for a column reference.
+ */
+ override def visitIdentityTransform(ctx: IdentityTransformContext): Transform = withOrigin(ctx) {
+ IdentityTransform(FieldReference(typedVisit[Seq[String]](ctx.multipartIdentifier())))
+ }
+
+ /**
+ * Create a named Transform from argument expressions.
+ */
+ override def visitApplyTransform(ctx: ApplyTransformContext): Transform = withOrigin(ctx) {
+ val args = toSeq(ctx.arguments).map(typedVisit[expressions.Expression])
+ ApplyTransform(ctx.transformName.getText, args)
+ }
+
+ /**
+ * Create a transform argument from a column reference or a constant.
+ */
+ override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression =
+ withOrigin(ctx) {
+ val reference = Option(ctx.multipartIdentifier())
+ .map(typedVisit[Seq[String]])
+ .map(FieldReference(_))
+ val literal = Option(ctx.constant)
+ .map(visitConstant)
+ .map(lit => LiteralValue(lit.value, lit.dataType))
+ reference
+ .orElse(literal)
+ .getOrElse(throw new IcebergParseException(s"Invalid transform argument", ctx))
+ }
+
+ /**
+ * Return a multi-part identifier as Seq[String].
+ */
+ override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] =
+ withOrigin(ctx) {
+ toSeq(ctx.parts).map(_.getText)
+ }
+
+ override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] =
+ withOrigin(ctx) {
+ toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
+ }
+
+ override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) {
+ visit(ctx.statement).asInstanceOf[LogicalPlan]
+ }
+
+ def visitConstant(ctx: ConstantContext): Literal = {
+ delegate.parseExpression(ctx.getText).asInstanceOf[Literal]
+ }
+
+ override def visitExpression(ctx: ExpressionContext): Expression = {
+ // reconstruct the SQL string and parse it using the main Spark parser
+ // while we can avoid the logic to build Spark expressions, we still have to parse them
+ // we cannot call ctx.getText directly since it will not render spaces correctly
+ // that's why we need to recurse down the tree in reconstructSqlString
+ val sqlString = reconstructSqlString(ctx)
+ delegate.parseExpression(sqlString)
+ }
+
+ private def reconstructSqlString(ctx: ParserRuleContext): String = {
+ toBuffer(ctx.children)
+ .map {
+ case c: ParserRuleContext => reconstructSqlString(c)
+ case t: TerminalNode => t.getText
+ }
+ .mkString(" ")
+ }
+
+ private def typedVisit[T](ctx: ParseTree): T = {
+ ctx.accept(this).asInstanceOf[T]
+ }
+}
+
+/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
+object IcebergParserUtils {
+
+ private[sql] def withOrigin[T](ctx: ParserRuleContext)(f: => T): T = {
+ val current = CurrentOrigin.get
+ CurrentOrigin.set(position(ctx.getStart))
+ try {
+ f
+ } finally {
+ CurrentOrigin.set(current)
+ }
+ }
+
+ private[sql] def position(token: Token): Origin = {
+ val opt = Option(token)
+ Origin(opt.map(_.getLine), opt.map(_.getCharPositionInLine))
+ }
+
+ /** Get the command which created the token. */
+ private[sql] def command(ctx: ParserRuleContext): String = {
+ val stream = ctx.getStart.getInputStream
+ stream.getText(Interval.of(0, stream.size() - 1))
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
new file mode 100644
index 0000000000..0a830dbd4f
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String])
+ extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"AddPartitionField ${table.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
new file mode 100644
index 0000000000..15b9083002
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+case class BranchOptions(
+ snapshotId: Option[Long],
+ numSnapshots: Option[Long],
+ snapshotRetain: Option[Long],
+ snapshotRefRetain: Option[Long])
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
new file mode 100644
index 0000000000..6900f6e8cc
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class CreateOrReplaceBranch(
+ table: Seq[String],
+ branch: String,
+ branchOptions: BranchOptions,
+ create: Boolean,
+ replace: Boolean,
+ ifNotExists: Boolean)
+ extends LeafCommand {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"CreateOrReplaceBranch branch: ${branch} for table: ${table.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
new file mode 100644
index 0000000000..957c68e7a5
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class CreateOrReplaceTag(
+ table: Seq[String],
+ tag: String,
+ tagOptions: TagOptions,
+ create: Boolean,
+ replace: Boolean,
+ ifNotExists: Boolean)
+ extends LeafCommand {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"CreateOrReplaceTag tag: ${tag} for table: ${table.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
new file mode 100644
index 0000000000..ed4f1f512b
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class DropBranch(table: Seq[String], branch: String, ifExists: Boolean) extends LeafCommand {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropBranch branch: ${branch} for table: ${table.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
new file mode 100644
index 0000000000..1a91806280
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class DropIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropIdentifierFields ${table.quoted} (${fields.quoted})"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
new file mode 100644
index 0000000000..ec952c8c71
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class DropPartitionField(table: Seq[String], transform: Transform) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropPartitionField ${table.quoted} ${transform.describe}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
new file mode 100644
index 0000000000..da69ca0383
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class DropTag(table: Seq[String], tag: String, ifExists: Boolean) extends LeafCommand {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropTag tag: ${tag} for table: ${table.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
new file mode 100644
index 0000000000..c2525369e7
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class ReplacePartitionField(
+ table: Seq[String],
+ transformFrom: Transform,
+ transformTo: Transform,
+ name: Option[String])
+ extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"ReplacePartitionField ${table.quoted} ${transformFrom.describe} " +
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
new file mode 100644
index 0000000000..8cd2c0ddad
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class SetIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override def simpleString(maxFields: Int): String = {
+ s"SetIdentifierFields ${table.quoted} (${fields.quoted})"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
new file mode 100644
index 0000000000..6afe1478d7
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical
+
+case class TagOptions(snapshotId: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
new file mode 100644
index 0000000000..84a00a4a9a
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical.views
+
+import org.apache.spark.sql.catalyst.analysis.AnalysisContext
+import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+// Align Iceberg's CreateIcebergView with Spark’s CreateViewCommand by extending AnalysisOnlyCommand.
+// The command’s children are analyzed then hidden, so the optimizer/planner won’t traverse the view body.
+case class CreateIcebergView(
+ child: LogicalPlan,
+ queryText: String,
+ query: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String] = Seq.empty,
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean,
+ rewritten: Boolean = false,
+ isAnalyzed: Boolean = false)
+ extends AnalysisOnlyCommand {
+
+ override def childrenToAnalyze: Seq[LogicalPlan] = child :: query :: Nil
+
+ override def markAsAnalyzed(analysisContext: AnalysisContext): LogicalPlan = {
+ copy(isAnalyzed = true)
+ }
+
+ override protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[LogicalPlan]): LogicalPlan = {
+ assert(!isAnalyzed)
+ copy(child = newChildren.head, query = newChildren.last)
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
new file mode 100644
index 0000000000..092b6b33fb
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical.views
+
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
+
+case class DropIcebergView(child: LogicalPlan, ifExists: Boolean) extends UnaryCommand {
+ override protected def withNewChildInternal(newChild: LogicalPlan): DropIcebergView =
+ copy(child = newChild)
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
new file mode 100644
index 0000000000..4d384e8577
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical.views
+
+import org.apache.spark.sql.catalyst.analysis.LeafNodeWithoutStats
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+
+case class ResolvedV2View(catalog: ViewCatalog, identifier: Identifier)
+ extends LeafNodeWithoutStats {
+ override def output: Seq[Attribute] = Nil
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
new file mode 100644
index 0000000000..cbfe23d94c
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.catalyst.plans.logical.views
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.ShowViews
+import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
+
+case class ShowIcebergViews(
+ namespace: LogicalPlan,
+ pattern: Option[String],
+ override val output: Seq[Attribute] = ShowViews.getOutputAttrs)
+ extends UnaryCommand {
+ override def child: LogicalPlan = namespace
+
+ override protected def withNewChildInternal(newChild: LogicalPlan): ShowIcebergViews =
+ copy(namespace = newChild)
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
new file mode 100644
index 0000000000..e28dcfb194
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class AddPartitionFieldExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ transform: Transform,
+ name: Option[String])
+ extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ iceberg.table
+ .updateSpec()
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
+ .commit()
+
+ case table =>
+ throw new UnsupportedOperationException(
+ s"Cannot add partition field to non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"AddPartitionField ${catalog.name}.${ident.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
new file mode 100644
index 0000000000..d6630e51ff
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.connector.catalog.ViewChange
+
+case class AlterV2ViewSetPropertiesExec(
+ catalog: ViewCatalog,
+ ident: Identifier,
+ properties: Map[String, String])
+ extends LeafV2CommandExec {
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ val changes = properties.map { case (property, value) =>
+ ViewChange.setProperty(property, value)
+ }.toSeq
+
+ catalog.alterView(ident, changes: _*)
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"AlterV2ViewSetProperties: ${ident}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
new file mode 100644
index 0000000000..aa57842f58
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.IcebergAnalysisException
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.connector.catalog.ViewChange
+
+case class AlterV2ViewUnsetPropertiesExec(
+ catalog: ViewCatalog,
+ ident: Identifier,
+ propertyKeys: Seq[String],
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ if (!ifExists) {
+ propertyKeys.filterNot(catalog.loadView(ident).properties.containsKey).foreach { property =>
+ throw new IcebergAnalysisException(s"Cannot remove property that is not set: '$property'")
+ }
+ }
+
+ val changes = propertyKeys.map(ViewChange.removeProperty)
+ catalog.alterView(ident, changes: _*)
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"AlterV2ViewUnsetProperties: ${ident}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
new file mode 100644
index 0000000000..baf985f53a
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.BranchOptions
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+
+case class CreateOrReplaceBranchExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ branch: String,
+ branchOptions: BranchOptions,
+ create: Boolean,
+ replace: Boolean,
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val snapshotId: java.lang.Long = branchOptions.snapshotId
+ .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId()))
+ .map(java.lang.Long.valueOf)
+ .orNull
+
+ val manageSnapshots = iceberg.table().manageSnapshots()
+ val refExists = null != iceberg.table().refs().get(branch)
+
+ def safeCreateBranch(): Unit = {
+ if (snapshotId == null) {
+ manageSnapshots.createBranch(branch)
+ } else {
+ manageSnapshots.createBranch(branch, snapshotId)
+ }
+ }
+
+ if (create && replace && !refExists) {
+ safeCreateBranch()
+ } else if (replace) {
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete replace branch operation on %s, main has no snapshot",
+ ident)
+ manageSnapshots.replaceBranch(branch, snapshotId)
+ } else {
+ if (refExists && ifNotExists) {
+ return Nil
+ }
+
+ safeCreateBranch()
+ }
+
+ if (branchOptions.numSnapshots.nonEmpty) {
+ manageSnapshots.setMinSnapshotsToKeep(branch, branchOptions.numSnapshots.get.toInt)
+ }
+
+ if (branchOptions.snapshotRetain.nonEmpty) {
+ manageSnapshots.setMaxSnapshotAgeMs(branch, branchOptions.snapshotRetain.get)
+ }
+
+ if (branchOptions.snapshotRefRetain.nonEmpty) {
+ manageSnapshots.setMaxRefAgeMs(branch, branchOptions.snapshotRefRetain.get)
+ }
+
+ manageSnapshots.commit()
+
+ case table =>
+ throw new UnsupportedOperationException(
+ s"Cannot create or replace branch on non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"CreateOrReplace branch: $branch for table: ${ident.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
new file mode 100644
index 0000000000..e486892614
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.TagOptions
+import org.apache.spark.sql.connector.catalog._
+
+case class CreateOrReplaceTagExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ tag: String,
+ tagOptions: TagOptions,
+ create: Boolean,
+ replace: Boolean,
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val snapshotId: java.lang.Long = tagOptions.snapshotId
+ .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId()))
+ .map(java.lang.Long.valueOf)
+ .orNull
+
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete create or replace tag operation on %s, main has no snapshot",
+ ident)
+
+ val manageSnapshot = iceberg.table.manageSnapshots()
+ val refExists = null != iceberg.table().refs().get(tag)
+
+ if (create && replace && !refExists) {
+ manageSnapshot.createTag(tag, snapshotId)
+ } else if (replace) {
+ manageSnapshot.replaceTag(tag, snapshotId)
+ } else {
+ if (refExists && ifNotExists) {
+ return Nil
+ }
+
+ manageSnapshot.createTag(tag, snapshotId)
+ }
+
+ if (tagOptions.snapshotRefRetain.nonEmpty) {
+ manageSnapshot.setMaxRefAgeMs(tag, tagOptions.snapshotRefRetain.get)
+ }
+
+ manageSnapshot.commit()
+
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot create tag to non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"Create tag: $tag for table: ${ident.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
new file mode 100644
index 0000000000..04f2b2af73
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.SupportsReplaceView
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.NoSuchViewException
+import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.connector.catalog.ViewInfo
+import org.apache.spark.sql.types.StructType
+import scala.jdk.CollectionConverters._
+
+case class CreateV2ViewExec(
+ catalog: ViewCatalog,
+ ident: Identifier,
+ queryText: String,
+ viewSchema: StructType,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String],
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean)
+ extends LeafV2CommandExec {
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name
+ val currentCatalog =
+ if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
+ val currentNamespace = session.sessionState.catalogManager.currentNamespace
+
+ val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION
+ val newProperties = properties ++
+ comment.map(ViewCatalog.PROP_COMMENT -> _) ++
+ Map(
+ ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
+ ViewCatalog.PROP_ENGINE_VERSION -> engineVersion)
+
+ if (replace) {
+ // CREATE OR REPLACE VIEW
+ catalog match {
+ case c: SupportsReplaceView =>
+ try {
+ replaceView(c, currentCatalog, currentNamespace, newProperties)
+ } catch {
+ // view might have been concurrently dropped during replace
+ case _: NoSuchViewException =>
+ replaceView(c, currentCatalog, currentNamespace, newProperties)
+ }
+ case _ =>
+ if (catalog.viewExists(ident)) {
+ catalog.dropView(ident)
+ }
+
+ createView(currentCatalog, currentNamespace, newProperties)
+ }
+ } else {
+ try {
+ // CREATE VIEW [IF NOT EXISTS]
+ createView(currentCatalog, currentNamespace, newProperties)
+ } catch {
+ case _: ViewAlreadyExistsException if allowExisting => // Ignore
+ }
+ }
+
+ Nil
+ }
+
+ private def replaceView(
+ supportsReplaceView: SupportsReplaceView,
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
+ supportsReplaceView.replaceView(
+ ident,
+ queryText,
+ currentCatalog,
+ currentNamespace,
+ viewSchema,
+ queryColumnNames.toArray,
+ columnAliases.toArray,
+ columnComments.map(c => c.orNull).toArray,
+ newProperties.asJava)
+ }
+
+ private def createView(
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
+ val viewInfo: ViewInfo = new ViewInfo(
+ ident,
+ queryText,
+ currentCatalog,
+ currentNamespace,
+ viewSchema,
+ queryColumnNames.toArray,
+ columnAliases.toArray,
+ columnComments.map(c => c.orNull).toArray,
+ newProperties.asJava)
+ catalog.createView(viewInfo)
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"CreateV2ViewExec: ${ident}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
new file mode 100644
index 0000000000..106734d207
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import scala.jdk.CollectionConverters._
+
+case class DescribeV2ViewExec(output: Seq[Attribute], view: View, isExtended: Boolean)
+ extends V2CommandExec
+ with LeafExecNode {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override protected def run(): Seq[InternalRow] = {
+ if (isExtended) {
+ (describeSchema :+ emptyRow) ++ describeExtended
+ } else {
+ describeSchema
+ }
+ }
+
+ private def describeSchema: Seq[InternalRow] =
+ view.schema().map { column =>
+ toCatalystRow(column.name, column.dataType.simpleString, column.getComment().getOrElse(""))
+ }
+
+ private def emptyRow: InternalRow = toCatalystRow("", "", "")
+
+ private def describeExtended: Seq[InternalRow] = {
+ val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
+ val properties: Map[String, String] =
+ view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
+ val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2).toIndexedSeq
+ val viewProperties = properties.toSeq
+ .sortBy(_._1)
+ .map { case (key, value) =>
+ s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
+ }
+ .mkString("[", ", ", "]")
+
+ // omitting view text here because it is shown as
+ // part of SHOW CREATE TABLE and can result in weird formatting in the DESCRIBE output
+ toCatalystRow("# Detailed View Information", "", "") ::
+ toCatalystRow("Comment", view.properties.getOrDefault(ViewCatalog.PROP_COMMENT, ""), "") ::
+ toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
+ toCatalystRow("View Query Output Columns", outputColumns, "") ::
+ toCatalystRow("View Properties", viewProperties, "") ::
+ toCatalystRow(
+ "Created By",
+ view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""),
+ "") ::
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DescribeV2ViewExec"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
new file mode 100644
index 0000000000..e7d9c7b70d
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+
+case class DropBranchExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ branch: String,
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val ref = iceberg.table().refs().get(branch)
+ if (ref != null || !ifExists) {
+ iceberg.table().manageSnapshots().removeBranch(branch).commit()
+ }
+
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot drop branch on non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropBranch branch: ${branch} for table: ${ident.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
new file mode 100644
index 0000000000..87b18594d5
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions
+import org.apache.iceberg.relocated.com.google.common.collect.Sets
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+
+case class DropIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val schema = iceberg.table.schema
+ val identifierFieldNames = Sets.newHashSet(schema.identifierFieldNames)
+
+ for (name <- fields) {
+ Preconditions.checkArgument(
+ schema.findField(name) != null,
+ "Cannot complete drop identifier fields operation: field %s not found",
+ name)
+ Preconditions.checkArgument(
+ identifierFieldNames.contains(name),
+ "Cannot complete drop identifier fields operation: %s is not an identifier field",
+ name)
+ identifierFieldNames.remove(name)
+ }
+
+ iceberg.table
+ .updateSchema()
+ .setIdentifierFields(identifierFieldNames)
+ .commit();
+ case table =>
+ throw new UnsupportedOperationException(
+ s"Cannot drop identifier fields in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
new file mode 100644
index 0000000000..db43263e0e
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.IdentityTransform
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class DropPartitionFieldExec(catalog: TableCatalog, ident: Identifier, transform: Transform)
+ extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val schema = iceberg.table.schema
+ transform match {
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
+ // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .commit()
+
+ case _ =>
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transform))
+ .commit()
+ }
+
+ case table =>
+ throw new UnsupportedOperationException(
+ s"Cannot drop partition field in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropPartitionField ${catalog.name}.${ident.quoted} ${transform.describe}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
new file mode 100644
index 0000000000..79b4a15255
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+
+case class DropTagExec(catalog: TableCatalog, ident: Identifier, tag: String, ifExists: Boolean)
+ extends LeafV2CommandExec {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val ref = iceberg.table().refs().get(tag)
+ if (ref != null || !ifExists) {
+ iceberg.table().manageSnapshots().removeTag(tag).commit()
+ }
+
+ case table =>
+ throw new UnsupportedOperationException(s"Cannot drop tag on non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropTag tag: ${tag} for table: ${ident.quoted}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
new file mode 100644
index 0000000000..6dd1188b78
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.NoSuchViewException
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+
+case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boolean)
+ extends LeafV2CommandExec {
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ val dropped = catalog.dropView(ident)
+ if (!dropped && !ifExists) {
+ throw new NoSuchViewException(ident)
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"DropV2View: ${ident}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
new file mode 100644
index 0000000000..da540f5891
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.SparkCatalog
+import org.apache.iceberg.spark.SparkSessionCatalog
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.analysis.IcebergAnalysisException
+import org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier
+import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace
+import org.apache.spark.sql.catalyst.expressions.PredicateHelper
+import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch
+import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag
+import org.apache.spark.sql.catalyst.plans.logical.DescribeRelation
+import org.apache.spark.sql.catalyst.plans.logical.DropBranch
+import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
+import org.apache.spark.sql.catalyst.plans.logical.DropTag
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.OrderAwareCoalesce
+import org.apache.spark.sql.catalyst.plans.logical.RenameTable
+import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
+import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
+import org.apache.spark.sql.catalyst.plans.logical.SetViewProperties
+import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
+import org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable
+import org.apache.spark.sql.catalyst.plans.logical.ShowTableProperties
+import org.apache.spark.sql.catalyst.plans.logical.UnsetViewProperties
+import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
+import org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView
+import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
+import org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews
+import org.apache.spark.sql.classic.Strategy
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.OrderAwareCoalesceExec
+import org.apache.spark.sql.execution.SparkPlan
+import scala.jdk.CollectionConverters._
+
+case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy with PredicateHelper {
+
+ override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
+ case AddPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform, name) =>
+ AddPartitionFieldExec(catalog, ident, transform, name) :: Nil
+
+ case CreateOrReplaceBranch(
+ IcebergCatalogAndIdentifier(catalog, ident),
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) =>
+ CreateOrReplaceBranchExec(
+ catalog,
+ ident,
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) :: Nil
+
+ case CreateOrReplaceTag(
+ IcebergCatalogAndIdentifier(catalog, ident),
+ tag,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists) =>
+ CreateOrReplaceTagExec(catalog, ident, tag, tagOptions, create, replace, ifNotExists) :: Nil
+
+ case DropBranch(IcebergCatalogAndIdentifier(catalog, ident), branch, ifExists) =>
+ DropBranchExec(catalog, ident, branch, ifExists) :: Nil
+
+ case DropTag(IcebergCatalogAndIdentifier(catalog, ident), tag, ifExists) =>
+ DropTagExec(catalog, ident, tag, ifExists) :: Nil
+
+ case DropPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform) =>
+ DropPartitionFieldExec(catalog, ident, transform) :: Nil
+
+ case ReplacePartitionField(
+ IcebergCatalogAndIdentifier(catalog, ident),
+ transformFrom,
+ transformTo,
+ name) =>
+ ReplacePartitionFieldExec(catalog, ident, transformFrom, transformTo, name) :: Nil
+
+ case SetIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
+ SetIdentifierFieldsExec(catalog, ident, fields) :: Nil
+
+ case DropIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
+ DropIdentifierFieldsExec(catalog, ident, fields) :: Nil
+
+ case SetWriteDistributionAndOrdering(
+ IcebergCatalogAndIdentifier(catalog, ident),
+ distributionMode,
+ ordering) =>
+ SetWriteDistributionAndOrderingExec(catalog, ident, distributionMode, ordering) :: Nil
+
+ case OrderAwareCoalesce(numPartitions, coalescer, child) =>
+ OrderAwareCoalesceExec(numPartitions, coalescer, planLater(child)) :: Nil
+
+ case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView @ true) =>
+ val newIdent = Spark3Util.catalogAndIdentifier(spark, newName.toList.asJava)
+ if (oldCatalog.name != newIdent.catalog().name()) {
+ throw new IcebergAnalysisException(
+ s"Cannot move view between catalogs: from=${oldCatalog.name} and to=${newIdent.catalog().name()}")
+ }
+ RenameV2ViewExec(oldCatalog, oldIdent, newIdent.identifier()) :: Nil
+
+ case DropIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), ifExists) =>
+ DropV2ViewExec(viewCatalog, ident, ifExists) :: Nil
+
+ case CreateIcebergView(
+ ResolvedIdentifier(viewCatalog: ViewCatalog, ident),
+ queryText,
+ query,
+ columnAliases,
+ columnComments,
+ queryColumnNames,
+ comment,
+ properties,
+ allowExisting,
+ replace,
+ _,
+ _) =>
+ CreateV2ViewExec(
+ catalog = viewCatalog,
+ ident = ident,
+ queryText = queryText,
+ columnAliases = columnAliases,
+ columnComments = columnComments,
+ queryColumnNames = queryColumnNames,
+ viewSchema = query.schema,
+ comment = comment,
+ properties = properties,
+ allowExisting = allowExisting,
+ replace = replace) :: Nil
+
+ case DescribeRelation(ResolvedV2View(catalog, ident), _, isExtended, output) =>
+ DescribeV2ViewExec(output, catalog.loadView(ident), isExtended) :: Nil
+
+ case ShowTableProperties(ResolvedV2View(catalog, ident), propertyKey, output) =>
+ ShowV2ViewPropertiesExec(output, catalog.loadView(ident), propertyKey) :: Nil
+
+ case ShowIcebergViews(ResolvedNamespace(catalog: ViewCatalog, namespace, _), pattern, output) =>
+ ShowV2ViewsExec(output, catalog, namespace, pattern) :: Nil
+
+ case ShowCreateTable(ResolvedV2View(catalog, ident), _, output) =>
+ ShowCreateV2ViewExec(output, catalog.loadView(ident)) :: Nil
+
+ case SetViewProperties(ResolvedV2View(catalog, ident), properties) =>
+ AlterV2ViewSetPropertiesExec(catalog, ident, properties) :: Nil
+
+ case UnsetViewProperties(ResolvedV2View(catalog, ident), propertyKeys, ifExists) =>
+ AlterV2ViewUnsetPropertiesExec(catalog, ident, propertyKeys, ifExists) :: Nil
+
+ case _ => Nil
+ }
+
+ private object IcebergCatalogAndIdentifier {
+ def unapply(identifier: Seq[String]): Option[(TableCatalog, Identifier)] = {
+ val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(spark, identifier.asJava)
+ catalogAndIdentifier.catalog match {
+ case icebergCatalog: SparkCatalog =>
+ Some((icebergCatalog, catalogAndIdentifier.identifier))
+ case icebergCatalog: SparkSessionCatalog[_] =>
+ Some((icebergCatalog, catalogAndIdentifier.identifier))
+ case _ =>
+ None
+ }
+ }
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
new file mode 100644
index 0000000000..5dada1cab0
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+
+case class RenameV2ViewExec(catalog: ViewCatalog, oldIdent: Identifier, newIdent: Identifier)
+ extends LeafV2CommandExec {
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.renameView(oldIdent, newIdent)
+
+ Seq.empty
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"RenameV2View ${oldIdent} to {newIdent}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
new file mode 100644
index 0000000000..00b998c49e
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.Spark3Util
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.expressions.FieldReference
+import org.apache.spark.sql.connector.expressions.IdentityTransform
+import org.apache.spark.sql.connector.expressions.Transform
+
+case class ReplacePartitionFieldExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ transformFrom: Transform,
+ transformTo: Transform,
+ name: Option[String])
+ extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val schema = iceberg.table.schema
+ transformFrom match {
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
+ // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
+
+ case _ =>
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transformFrom))
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
+ }
+
+ case table =>
+ throw new UnsupportedOperationException(
+ s"Cannot replace partition field in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ReplacePartitionField ${catalog.name}.${ident.quoted} ${transformFrom.describe} " +
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
new file mode 100644
index 0000000000..50c53473ab
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+import scala.jdk.CollectionConverters._
+
+case class SetIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ iceberg.table
+ .updateSchema()
+ .setIdentifierFields(fields.asJava)
+ .commit();
+ case table =>
+ throw new UnsupportedOperationException(
+ s"Cannot set identifier fields in non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"SetIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
new file mode 100644
index 0000000000..9a10949d5e
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.iceberg.DistributionMode
+import org.apache.iceberg.NullOrder
+import org.apache.iceberg.SortDirection
+import org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE
+import org.apache.iceberg.expressions.Term
+import org.apache.iceberg.spark.SparkUtil
+import org.apache.iceberg.spark.source.SparkTable
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.catalog.TableCatalog
+
+case class SetWriteDistributionAndOrderingExec(
+ catalog: TableCatalog,
+ ident: Identifier,
+ distributionMode: Option[DistributionMode],
+ sortOrder: Seq[(Term, SortDirection, NullOrder)])
+ extends LeafV2CommandExec {
+
+ import CatalogV2Implicits._
+
+ override lazy val output: Seq[Attribute] = Nil
+
+ override protected def run(): Seq[InternalRow] = {
+ catalog.loadTable(ident) match {
+ case iceberg: SparkTable =>
+ val txn = iceberg.table.newTransaction()
+
+ val orderBuilder = txn.replaceSortOrder().caseSensitive(SparkUtil.caseSensitive(session))
+ sortOrder.foreach {
+ case (term, SortDirection.ASC, nullOrder) =>
+ orderBuilder.asc(term, nullOrder)
+ case (term, SortDirection.DESC, nullOrder) =>
+ orderBuilder.desc(term, nullOrder)
+ }
+ orderBuilder.commit()
+
+ distributionMode.foreach { mode =>
+ txn
+ .updateProperties()
+ .set(WRITE_DISTRIBUTION_MODE, mode.modeName())
+ .commit()
+ }
+
+ txn.commitTransaction()
+
+ case table =>
+ throw new UnsupportedOperationException(
+ s"Cannot set write order of non-Iceberg table: $table")
+ }
+
+ Nil
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ val tableIdent = s"${catalog.name}.${ident.quoted}"
+ val order = sortOrder
+ .map { case (term, direction, nullOrder) =>
+ s"$term $direction $nullOrder"
+ }
+ .mkString(", ")
+ s"SetWriteDistributionAndOrdering $tableIdent $distributionMode $order"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
new file mode 100644
index 0000000000..07ac4aeda8
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import scala.jdk.CollectionConverters._
+
+case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
+ extends V2CommandExec
+ with LeafExecNode {
+
+ override protected def run(): Seq[InternalRow] = {
+ val builder = new StringBuilder
+ builder ++= s"CREATE VIEW ${view.name} "
+ showColumns(view, builder)
+ showComment(view, builder)
+ showProperties(view, builder)
+ builder ++= s"AS\n${view.query}\n"
+
+ Seq(toCatalystRow(builder.toString))
+ }
+
+ private def showColumns(view: View, builder: StringBuilder): Unit = {
+ val columns = concatByMultiLines(
+ view
+ .schema()
+ .fields
+ .map(x => s"${x.name}${x.getComment().map(c => s" COMMENT '$c'").getOrElse("")}"))
+ builder ++= columns
+ }
+
+ private def showComment(view: View, builder: StringBuilder): Unit = {
+ Option(view.properties.get(ViewCatalog.PROP_COMMENT))
+ .map("COMMENT '" + escapeSingleQuotedString(_) + "'\n")
+ .foreach(builder.append)
+ }
+
+ private def showProperties(view: View, builder: StringBuilder): Unit = {
+ val showProps = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
+ if (showProps.nonEmpty) {
+ val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map { case (key, value) =>
+ s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
+ }
+
+ builder ++= "TBLPROPERTIES "
+ builder ++= concatByMultiLines(props)
+ }
+ }
+
+ private def concatByMultiLines(iter: Iterable[String]): String = {
+ iter.mkString("(\n ", ",\n ", ")\n")
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ShowCreateV2ViewExec"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
new file mode 100644
index 0000000000..ace43eb6c0
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.connector.catalog.View
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import scala.jdk.CollectionConverters._
+
+case class ShowV2ViewPropertiesExec(output: Seq[Attribute], view: View, propertyKey: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
+
+ override protected def run(): Seq[InternalRow] = {
+ propertyKey match {
+ case Some(p) =>
+ val propValue = properties.getOrElse(p, s"View ${view.name()} does not have property: $p")
+ Seq(toCatalystRow(p, propValue))
+ case None =>
+ properties.map { case (k, v) =>
+ toCatalystRow(k, v)
+ }.toSeq
+ }
+ }
+
+ private def properties = {
+ view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ShowV2ViewPropertiesExec"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
new file mode 100644
index 0000000000..4e7700b439
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.connector.catalog.ViewCatalog
+import org.apache.spark.sql.execution.LeafExecNode
+import org.apache.spark.sql.internal.SQLConf
+import scala.collection.mutable.ArrayBuffer
+
+case class ShowV2ViewsExec(
+ output: Seq[Attribute],
+ catalog: ViewCatalog,
+ namespace: Seq[String],
+ pattern: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
+
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+ override protected def run(): Seq[InternalRow] = {
+ val rows = new ArrayBuffer[InternalRow]()
+
+ // handle GLOBAL VIEWS
+ val globalTemp = SQLConf.get.globalTempDatabase
+ if (namespace.nonEmpty && globalTemp == namespace.head) {
+ pattern
+ .map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
+ .getOrElse(session.sessionState.catalog.globalTempViewManager.listViewNames("*"))
+ .map(name => rows += toCatalystRow(globalTemp, name, true))
+ } else {
+ val views = catalog.listViews(namespace: _*)
+ views.map { view =>
+ if (pattern.map(StringUtils.filterPattern(Seq(view.name()), _).nonEmpty).getOrElse(true)) {
+ rows += toCatalystRow(view.namespace().quoted, view.name(), false)
+ }
+ }
+ }
+
+ // include TEMP VIEWS
+ pattern
+ .map(p => session.sessionState.catalog.listLocalTempViews(p))
+ .getOrElse(session.sessionState.catalog.listLocalTempViews("*"))
+ .map(v => rows += toCatalystRow(v.database.toArray.quoted, v.table, true))
+
+ rows.toSeq
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ s"ShowV2ViewsExec"
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/TestExtendedParser.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/TestExtendedParser.java
new file mode 100644
index 0000000000..bfcb5af235
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/TestExtendedParser.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.List;
+import org.apache.iceberg.NullOrder;
+import org.apache.iceberg.SortDirection;
+import org.apache.iceberg.expressions.Term;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.parser.AbstractSqlParser;
+import org.apache.spark.sql.catalyst.parser.AstBuilder;
+import org.apache.spark.sql.catalyst.parser.ParserInterface;
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class TestExtendedParser {
+
+ private static SparkSession spark;
+ private static final String SQL_PARSER_FIELD = "sqlParser";
+ private ParserInterface originalParser;
+
+ @BeforeAll
+ public static void before() {
+ spark = SparkSession.builder().master("local").appName("TestExtendedParser").getOrCreate();
+ }
+
+ @AfterAll
+ public static void after() {
+ if (spark != null) {
+ spark.stop();
+ }
+ }
+
+ @BeforeEach
+ public void saveOriginalParser() throws Exception {
+ Class<?> clazz = spark.sessionState().getClass();
+ Field parserField = null;
+ while (clazz != null && parserField == null) {
+ try {
+ parserField = clazz.getDeclaredField(SQL_PARSER_FIELD);
+ } catch (NoSuchFieldException e) {
+ clazz = clazz.getSuperclass();
+ }
+ }
+ parserField.setAccessible(true);
+ originalParser = (ParserInterface) parserField.get(spark.sessionState());
+ }
+
+ @AfterEach
+ public void restoreOriginalParser() throws Exception {
+ setSessionStateParser(spark.sessionState(), originalParser);
+ }
+
+ /**
+ * Tests that the Iceberg extended SQL parser can correctly parse a sort order string and return
+ * the expected RawOrderField.
+ *
+ * @throws Exception if reflection access fails
+ */
+ @Test
+ public void testParseSortOrderWithRealIcebergExtendedParser() throws Exception {
+ ParserInterface origParser = null;
+ Class<?> clazz = spark.sessionState().getClass();
+ while (clazz != null && origParser == null) {
+ try {
+ Field parserField = clazz.getDeclaredField(SQL_PARSER_FIELD);
+ parserField.setAccessible(true);
+ origParser = (ParserInterface) parserField.get(spark.sessionState());
+ } catch (NoSuchFieldException e) {
+ clazz = clazz.getSuperclass();
+ }
+ }
+ assertThat(origParser).isNotNull();
+
+ IcebergSparkSqlExtensionsParser icebergParser = new IcebergSparkSqlExtensionsParser(origParser);
+
+ setSessionStateParser(spark.sessionState(), icebergParser);
+
+ List<ExtendedParser.RawOrderField> fields =
+ ExtendedParser.parseSortOrder(spark, "id ASC NULLS FIRST");
+
+ assertThat(fields).isNotEmpty();
+ ExtendedParser.RawOrderField first = fields.get(0);
+ assertThat(first.direction()).isEqualTo(SortDirection.ASC);
+ assertThat(first.nullOrder()).isEqualTo(NullOrder.NULLS_FIRST);
+ }
+
+ /**
+ * Tests that parseSortOrder can find and use an ExtendedParser that is wrapped inside another
+ * ParserInterface implementation.
+ *
+ * @throws Exception if reflection access fails
+ */
+ @Test
+ public void testParseSortOrderFindsNestedExtendedParser() throws Exception {
+ ExtendedParser icebergParser = mock(ExtendedParser.class);
+
+ ExtendedParser.RawOrderField field =
+ new ExtendedParser.RawOrderField(
+ mock(Term.class), SortDirection.ASC, NullOrder.NULLS_FIRST);
+ List<ExtendedParser.RawOrderField> expected = Collections.singletonList(field);
+
+ when(icebergParser.parseSortOrder("id ASC NULLS FIRST")).thenReturn(expected);
+
+ ParserInterface wrapper = new WrapperParser(icebergParser);
+
+ setSessionStateParser(spark.sessionState(), wrapper);
+
+ List<ExtendedParser.RawOrderField> result =
+ ExtendedParser.parseSortOrder(spark, "id ASC NULLS FIRST");
+ assertThat(result).isSameAs(expected);
+
+ verify(icebergParser).parseSortOrder("id ASC NULLS FIRST");
+ }
+
+ /**
+ * Tests that parseSortOrder throws an exception if no ExtendedParser instance can be found in the
+ * parser chain.
+ *
+ * @throws Exception if reflection access fails
+ */
+ @Test
+ public void testParseSortOrderThrowsWhenNoExtendedParserFound() throws Exception {
+ ParserInterface dummy = mock(ParserInterface.class);
+ setSessionStateParser(spark.sessionState(), dummy);
+
+ assertThatThrownBy(() -> ExtendedParser.parseSortOrder(spark, "id ASC"))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Iceberg ExtendedParser");
+ }
+
+ /**
+ * Tests that parseSortOrder can find an ExtendedParser in a parent class field of the parser.
+ *
+ * @throws Exception if reflection access fails
+ */
+ @Test
+ public void testParseSortOrderFindsExtendedParserInParentClassField() throws Exception {
+ ExtendedParser icebergParser = mock(ExtendedParser.class);
+ ExtendedParser.RawOrderField field =
+ new ExtendedParser.RawOrderField(
+ mock(Term.class), SortDirection.ASC, NullOrder.NULLS_FIRST);
+ List<ExtendedParser.RawOrderField> expected = Collections.singletonList(field);
+ when(icebergParser.parseSortOrder("id ASC NULLS FIRST")).thenReturn(expected);
+ ParserInterface parser = new GrandChildParser(icebergParser);
+ setSessionStateParser(spark.sessionState(), parser);
+
+ List<ExtendedParser.RawOrderField> result =
+ ExtendedParser.parseSortOrder(spark, "id ASC NULLS FIRST");
+ assertThat(result).isSameAs(expected);
+ verify(icebergParser).parseSortOrder("id ASC NULLS FIRST");
+ }
+
+ private static void setSessionStateParser(Object sessionState, ParserInterface parser)
+ throws Exception {
+ Class<?> clazz = sessionState.getClass();
+ Field targetField = null;
+ while (clazz != null && targetField == null) {
+ try {
+ targetField = clazz.getDeclaredField(SQL_PARSER_FIELD);
+ } catch (NoSuchFieldException e) {
+ clazz = clazz.getSuperclass();
+ }
+ }
+ if (targetField == null) {
+ throw new IllegalStateException(
+ "No suitable sqlParser field found in sessionState class hierarchy!");
+ }
+ targetField.setAccessible(true);
+ targetField.set(sessionState, parser);
+ }
+
+ private static class WrapperParser extends AbstractSqlParser {
+ private final ParserInterface delegate;
+ private String name;
+
+ WrapperParser(ParserInterface delegate) {
+ this.delegate = delegate;
+ this.name = "delegate";
+ }
+
+ public ParserInterface getDelegate() {
+ return delegate;
+ }
+
+ @Override
+ public AstBuilder astBuilder() {
+ return null;
+ }
+ }
+
+ private static class ChildParser extends WrapperParser {
+ ChildParser(ParserInterface parent) {
+ super(parent);
+ }
+ }
+
+ private static class GrandChildParser extends ChildParser {
+ GrandChildParser(ParserInterface parent) {
+ super(parent);
+ }
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
new file mode 100644
index 0000000000..8918dfec65
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import java.util.Objects;
+
+public class Employee {
+ private Integer id;
+ private String dep;
+
+ public Employee() {}
+
+ public Employee(Integer id, String dep) {
+ this.id = id;
+ this.dep = dep;
+ }
+
+ public Integer getId() {
+ return id;
+ }
+
+ public void setId(Integer id) {
+ this.id = id;
+ }
+
+ public String getDep() {
+ return dep;
+ }
+
+ public void setDep(String dep) {
+ this.dep = dep;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ Employee employee = (Employee) other;
+ return Objects.equals(id, employee.id) && Objects.equals(dep, employee.dep);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, dep);
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
new file mode 100644
index 0000000000..796c47b545
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS;
+
+import java.net.InetAddress;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.hive.HiveCatalog;
+import org.apache.iceberg.hive.TestHiveMetastore;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.spark.CatalogTestBase;
+import org.apache.iceberg.spark.TestBase;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.internal.SQLConf;
+import org.junit.jupiter.api.BeforeAll;
+
+public abstract class ExtensionsTestBase extends CatalogTestBase {
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @BeforeAll
+ public static void startMetastoreAndSpark() {
+ TestBase.metastore = new TestHiveMetastore();
+ metastore.start();
+ TestBase.hiveConf = metastore.hiveConf();
+
+ TestBase.spark.stop();
+
+ TestBase.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress())
+ .config("spark.testing", "true")
+ .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic")
+ .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
+ .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname))
+ .config("spark.sql.shuffle.partitions", "4")
+ .config("spark.sql.hive.metastorePartitionPruningFallbackOnException", "true")
+ .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true")
+ .config(
+ SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean()))
+ .enableHiveSupport()
+ .getOrCreate();
+
+ TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+ TestBase.catalog =
+ (HiveCatalog)
+ CatalogUtil.loadCatalog(
+ HiveCatalog.class.getName(), "hive", ImmutableMap.of(), hiveConf);
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java
new file mode 100644
index 0000000000..de4acd74a7
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.UUID;
+import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.PositionOutputStream;
+
+public class ProcedureUtil {
+
+ private ProcedureUtil() {}
+
+ static PartitionStatisticsFile writePartitionStatsFile(
+ long snapshotId, String statsLocation, FileIO fileIO) {
+ PositionOutputStream positionOutputStream;
+ try {
+ positionOutputStream = fileIO.newOutputFile(statsLocation).create();
+ positionOutputStream.close();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ return ImmutableGenericPartitionStatisticsFile.builder()
+ .snapshotId(snapshotId)
+ .fileSizeInBytes(42L)
+ .path(statsLocation)
+ .build();
+ }
+
+ static String statsFileLocation(String tableLocation) {
+ String statsFileName = "stats-file-" + UUID.randomUUID();
+ return tableLocation.replaceFirst("file:", "") + "/metadata/" + statsFileName;
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkPlanUtil.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkPlanUtil.java
new file mode 100644
index 0000000000..830d07d86e
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkPlanUtil.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static scala.collection.JavaConverters.seqAsJavaListConverter;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.spark.sql.catalyst.expressions.Expression;
+import org.apache.spark.sql.execution.CommandResultExec;
+import org.apache.spark.sql.execution.SparkPlan;
+import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper;
+import org.apache.spark.sql.execution.datasources.v2.BatchScanExec;
+import scala.PartialFunction;
+import scala.collection.Seq;
+
+public class SparkPlanUtil {
+
+ private static final AdaptiveSparkPlanHelper SPARK_HELPER = new AdaptiveSparkPlanHelper() {};
+
+ private SparkPlanUtil() {}
+
+ public static List<SparkPlan> collectLeaves(SparkPlan plan) {
+ return toJavaList(SPARK_HELPER.collectLeaves(actualPlan(plan)));
+ }
+
+ public static List<SparkPlan> collectBatchScans(SparkPlan plan) {
+ List<SparkPlan> leaves = collectLeaves(plan);
+ return leaves.stream()
+ .filter(scan -> scan instanceof BatchScanExec)
+ .collect(Collectors.toList());
+ }
+
+ private static SparkPlan actualPlan(SparkPlan plan) {
+ if (plan instanceof CommandResultExec) {
+ return ((CommandResultExec) plan).commandPhysicalPlan();
+ } else {
+ return plan;
+ }
+ }
+
+ public static List<Expression> collectExprs(
+ SparkPlan sparkPlan, Predicate<Expression> predicate) {
+ Seq<List<Expression>> seq =
+ SPARK_HELPER.collect(
+ sparkPlan,
+ new PartialFunction<SparkPlan, List<Expression>>() {
+ @Override
+ public List<Expression> apply(SparkPlan plan) {
+ List<Expression> exprs = Lists.newArrayList();
+
+ for (Expression expr : toJavaList(plan.expressions())) {
+ exprs.addAll(collectExprs(expr, predicate));
+ }
+
+ return exprs;
+ }
+
+ @Override
+ public boolean isDefinedAt(SparkPlan plan) {
+ return true;
+ }
+ });
+ return toJavaList(seq).stream().flatMap(Collection::stream).collect(Collectors.toList());
+ }
+
+ private static List<Expression> collectExprs(
+ Expression expression, Predicate<Expression> predicate) {
+ Seq<Expression> seq =
+ expression.collect(
+ new PartialFunction<Expression, Expression>() {
+ @Override
+ public Expression apply(Expression expr) {
+ return expr;
+ }
+
+ @Override
+ public boolean isDefinedAt(Expression expr) {
+ return predicate.test(expr);
+ }
+ });
+ return toJavaList(seq);
+ }
+
+ private static <T> List<T> toJavaList(Seq<T> seq) {
+ return seqAsJavaListConverter(seq).asJava();
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
new file mode 100644
index 0000000000..b5d6415763
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
@@ -0,0 +1,461 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.apache.iceberg.DataOperations.DELETE;
+import static org.apache.iceberg.DataOperations.OVERWRITE;
+import static org.apache.iceberg.PlanningMode.DISTRIBUTED;
+import static org.apache.iceberg.PlanningMode.LOCAL;
+import static org.apache.iceberg.SnapshotSummary.ADDED_DELETE_FILES_PROP;
+import static org.apache.iceberg.SnapshotSummary.ADDED_DVS_PROP;
+import static org.apache.iceberg.SnapshotSummary.ADDED_FILES_PROP;
+import static org.apache.iceberg.SnapshotSummary.ADD_POS_DELETE_FILES_PROP;
+import static org.apache.iceberg.SnapshotSummary.CHANGED_PARTITION_COUNT_PROP;
+import static org.apache.iceberg.SnapshotSummary.DELETED_FILES_PROP;
+import static org.apache.iceberg.TableProperties.DATA_PLANNING_MODE;
+import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
+import static org.apache.iceberg.TableProperties.DELETE_PLANNING_MODE;
+import static org.apache.iceberg.TableProperties.FORMAT_VERSION;
+import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED;
+import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED;
+import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE;
+import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PlanningMode;
+import org.apache.iceberg.RowLevelOperationMode;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.data.parquet.GenericParquetWriter;
+import org.apache.iceberg.deletes.DeleteGranularity;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.spark.SparkCatalog;
+import org.apache.iceberg.spark.SparkSessionCatalog;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoder;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.execution.SparkPlan;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase {
+
+ private static final Random RANDOM = ThreadLocalRandom.current();
+
+ @Parameter(index = 3)
+ protected FileFormat fileFormat;
+
+ @Parameter(index = 4)
+ protected boolean vectorized;
+
+ @Parameter(index = 5)
+ protected String distributionMode;
+
+ @Parameter(index = 6)
+ protected boolean fanoutEnabled;
+
+ @Parameter(index = 7)
+ protected String branch;
+
+ @Parameter(index = 8)
+ protected PlanningMode planningMode;
+
+ @Parameter(index = 9)
+ protected int formatVersion;
+
+ @Parameters(
+ name =
+ "catalogName = {0}, implementation = {1}, config = {2},"
+ + " format = {3}, vectorized = {4}, distributionMode = {5},"
+ + " fanout = {6}, branch = {7}, planningMode = {8}, formatVersion = {9}")
+ public static Object[][] parameters() {
+ return new Object[][] {
+ {
+ "testhive",
+ SparkCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default"),
+ FileFormat.ORC,
+ true,
+ WRITE_DISTRIBUTION_MODE_NONE,
+ true,
+ SnapshotRef.MAIN_BRANCH,
+ LOCAL,
+ 2
+ },
+ {
+ "testhive",
+ SparkCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default"),
+ FileFormat.PARQUET,
+ true,
+ WRITE_DISTRIBUTION_MODE_NONE,
+ false,
+ "test",
+ DISTRIBUTED,
+ 2
+ },
+ {
+ "testhadoop",
+ SparkCatalog.class.getName(),
+ ImmutableMap.of("type", "hadoop"),
+ FileFormat.PARQUET,
+ RANDOM.nextBoolean(),
+ WRITE_DISTRIBUTION_MODE_HASH,
+ true,
+ null,
+ LOCAL,
+ 2
+ },
+ {
+ "spark_catalog",
+ SparkSessionCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default",
+ "clients", "1",
+ "parquet-enabled", "false",
+ "cache-enabled",
+ "false" // Spark will delete tables using v1, leaving the cache out of sync
+ ),
+ FileFormat.AVRO,
+ false,
+ WRITE_DISTRIBUTION_MODE_RANGE,
+ false,
+ "test",
+ DISTRIBUTED,
+ 2
+ },
+ {
+ "testhadoop",
+ SparkCatalog.class.getName(),
+ ImmutableMap.of("type", "hadoop"),
+ FileFormat.PARQUET,
+ RANDOM.nextBoolean(),
+ WRITE_DISTRIBUTION_MODE_HASH,
+ true,
+ null,
+ LOCAL,
+ 3
+ },
+ {
+ "spark_catalog",
+ SparkSessionCatalog.class.getName(),
+ ImmutableMap.of(
+ "type",
+ "hive",
+ "default-namespace",
+ "default",
+ "clients",
+ "1",
+ "parquet-enabled",
+ "false",
+ "cache-enabled",
+ "false" // Spark will delete tables using v1, leaving the cache out of sync
+ ),
+ FileFormat.AVRO,
+ false,
+ WRITE_DISTRIBUTION_MODE_RANGE,
+ false,
+ "test",
+ DISTRIBUTED,
+ 3
+ },
+ };
+ }
+
+ protected abstract Map<String, String> extraTableProperties();
+
+ protected void initTable() {
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES('%s' '%s', '%s' '%s', '%s' '%s', '%s' '%s', '%s' '%s', '%s' '%s')",
+ tableName,
+ DEFAULT_FILE_FORMAT,
+ fileFormat,
+ WRITE_DISTRIBUTION_MODE,
+ distributionMode,
+ SPARK_WRITE_PARTITIONED_FANOUT_ENABLED,
+ String.valueOf(fanoutEnabled),
+ DATA_PLANNING_MODE,
+ planningMode.modeName(),
+ DELETE_PLANNING_MODE,
+ planningMode.modeName(),
+ FORMAT_VERSION,
+ formatVersion);
+
+ switch (fileFormat) {
+ case PARQUET:
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES('%s' '%b')",
+ tableName, PARQUET_VECTORIZATION_ENABLED, vectorized);
+ break;
+ case ORC:
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES('%s' '%b')",
+ tableName, ORC_VECTORIZATION_ENABLED, vectorized);
+ break;
+ case AVRO:
+ assertThat(vectorized).isFalse();
+ break;
+ }
+
+ Map<String, String> props = extraTableProperties();
+ props.forEach(
+ (prop, value) -> {
+ sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')", tableName, prop, value);
+ });
+ }
+
+ protected void createAndInitTable(String schema) {
+ createAndInitTable(schema, null);
+ }
+
+ protected void createAndInitTable(String schema, String jsonData) {
+ createAndInitTable(schema, "", jsonData);
+ }
+
+ protected void createAndInitTable(String schema, String partitioning, String jsonData) {
+ sql("CREATE TABLE %s (%s) USING iceberg %s", tableName, schema, partitioning);
+ initTable();
+
+ if (jsonData != null) {
+ try {
+ Dataset<Row> ds = toDS(schema, jsonData);
+ ds.coalesce(1).writeTo(tableName).append();
+ createBranchIfNeeded();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException("Failed to write data", e);
+ }
+ }
+ }
+
+ protected void append(String table, String jsonData) {
+ append(table, null, jsonData);
+ }
+
+ protected void append(String table, String schema, String jsonData) {
+ try {
+ Dataset<Row> ds = toDS(schema, jsonData);
+ ds.coalesce(1).writeTo(table).append();
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException("Failed to write data", e);
+ }
+ }
+
+ protected void createOrReplaceView(String name, String jsonData) {
+ createOrReplaceView(name, null, jsonData);
+ }
+
+ protected void createOrReplaceView(String name, String schema, String jsonData) {
+ Dataset<Row> ds = toDS(schema, jsonData);
+ ds.createOrReplaceTempView(name);
+ }
+
+ protected <T> void createOrReplaceView(String name, List<T> data, Encoder<T> encoder) {
+ spark.createDataset(data, encoder).createOrReplaceTempView(name);
+ }
+
+ private Dataset<Row> toDS(String schema, String jsonData) {
+ List<String> jsonRows =
+ Arrays.stream(jsonData.split("\n"))
+ .filter(str -> !str.trim().isEmpty())
+ .collect(Collectors.toList());
+ Dataset<String> jsonDS = spark.createDataset(jsonRows, Encoders.STRING());
+
+ if (schema != null) {
+ return spark.read().schema(schema).json(jsonDS);
+ } else {
+ return spark.read().json(jsonDS);
+ }
+ }
+
+ protected void validateDelete(
+ Snapshot snapshot, String changedPartitionCount, String deletedDataFiles) {
+ validateSnapshot(snapshot, DELETE, changedPartitionCount, deletedDataFiles, null, null);
+ }
+
+ protected void validateCopyOnWrite(
+ Snapshot snapshot,
+ String changedPartitionCount,
+ String deletedDataFiles,
+ String addedDataFiles) {
+ String operation = null == addedDataFiles && null != deletedDataFiles ? DELETE : OVERWRITE;
+ validateSnapshot(
+ snapshot, operation, changedPartitionCount, deletedDataFiles, null, addedDataFiles);
+ }
+
+ protected void validateMergeOnRead(
+ Snapshot snapshot,
+ String changedPartitionCount,
+ String addedDeleteFiles,
+ String addedDataFiles) {
+ String operation = null == addedDataFiles && null != addedDeleteFiles ? DELETE : OVERWRITE;
+ validateSnapshot(
+ snapshot, operation, changedPartitionCount, null, addedDeleteFiles, addedDataFiles);
+ }
+
+ protected void validateSnapshot(
+ Snapshot snapshot,
+ String operation,
+ String changedPartitionCount,
+ String deletedDataFiles,
+ String addedDeleteFiles,
+ String addedDataFiles) {
+ assertThat(snapshot.operation()).as("Operation must match").isEqualTo(operation);
+ validateProperty(snapshot, CHANGED_PARTITION_COUNT_PROP, changedPartitionCount);
+ validateProperty(snapshot, DELETED_FILES_PROP, deletedDataFiles);
+ validateProperty(snapshot, ADDED_DELETE_FILES_PROP, addedDeleteFiles);
+ validateProperty(snapshot, ADDED_FILES_PROP, addedDataFiles);
+ if (formatVersion >= 3) {
+ validateProperty(snapshot, ADDED_DVS_PROP, addedDeleteFiles);
+ assertThat(snapshot.summary()).doesNotContainKey(ADD_POS_DELETE_FILES_PROP);
+ }
+ }
+
+ protected void validateProperty(Snapshot snapshot, String property, Set<String> expectedValues) {
+ String actual = snapshot.summary().get(property);
+ assertThat(actual)
+ .as(
+ "Snapshot property "
+ + property
+ + " has unexpected value, actual = "
+ + actual
+ + ", expected one of : "
+ + String.join(",", expectedValues))
+ .isIn(expectedValues);
+ }
+
+ protected void validateProperty(Snapshot snapshot, String property, String expectedValue) {
+ if (null == expectedValue) {
+ assertThat(snapshot.summary()).doesNotContainKey(property);
+ } else {
+ assertThat(snapshot.summary())
+ .as("Snapshot property " + property + " has unexpected value.")
+ .containsEntry(property, expectedValue);
+ }
+ }
+
+ protected void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected DataFile writeDataFile(Table table, List<GenericRecord> records) {
+ try {
+ OutputFile file =
+ Files.localOutput(
+ temp.resolve(fileFormat.addExtension(UUID.randomUUID().toString())).toFile());
+
+ DataWriter<GenericRecord> dataWriter =
+ Parquet.writeData(file)
+ .forTable(table)
+ .createWriterFunc(GenericParquetWriter::create)
+ .overwrite()
+ .build();
+
+ try {
+ for (GenericRecord record : records) {
+ dataWriter.write(record);
+ }
+ } finally {
+ dataWriter.close();
+ }
+
+ return dataWriter.toDataFile();
+
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ protected String commitTarget() {
+ return branch == null ? tableName : String.format("%s.branch_%s", tableName, branch);
+ }
+
+ @Override
+ protected String selectTarget() {
+ return branch == null ? tableName : String.format("%s VERSION AS OF '%s'", tableName, branch);
+ }
+
+ protected void createBranchIfNeeded() {
+ if (branch != null && !branch.equals(SnapshotRef.MAIN_BRANCH)) {
+ sql("ALTER TABLE %s CREATE BRANCH %s", tableName, branch);
+ }
+ }
+
+ // ORC currently does not support vectorized reads with deletes
+ protected boolean supportsVectorization() {
+ return vectorized && (isParquet() || isCopyOnWrite());
+ }
+
+ private boolean isParquet() {
+ return fileFormat.equals(FileFormat.PARQUET);
+ }
+
+ private boolean isCopyOnWrite() {
+ return extraTableProperties().containsValue(RowLevelOperationMode.COPY_ON_WRITE.modeName());
+ }
+
+ protected void assertAllBatchScansVectorized(SparkPlan plan) {
+ List<SparkPlan> batchScans = SparkPlanUtil.collectBatchScans(plan);
+ assertThat(batchScans).hasSizeGreaterThan(0).allMatch(SparkPlan::supportsColumnar);
+ }
+
+ protected void createTableWithDeleteGranularity(
+ String schema, String partitionedBy, DeleteGranularity deleteGranularity) {
+ createAndInitTable(schema, partitionedBy, null /* empty */);
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')",
+ tableName, TableProperties.DELETE_GRANULARITY, deleteGranularity);
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
new file mode 100644
index 0000000000..29993380b5
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
@@ -0,0 +1,1479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assumptions.assumeThat;
+
+import java.io.File;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumWriter;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.Spark3Util;
+import org.apache.iceberg.spark.SparkCatalogConfig;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.parser.ParseException;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestAddFilesProcedure extends ExtensionsTestBase {
+
+ @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}, formatVersion = {3}")
+ public static Object[][] parameters() {
+ return new Object[][] {
+ {
+ SparkCatalogConfig.HIVE.catalogName(),
+ SparkCatalogConfig.HIVE.implementation(),
+ SparkCatalogConfig.HIVE.properties(),
+ 1
+ },
+ {
+ SparkCatalogConfig.HADOOP.catalogName(),
+ SparkCatalogConfig.HADOOP.implementation(),
+ SparkCatalogConfig.HADOOP.properties(),
+ 2
+ },
+ {
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
+ 2
+ }
+ };
+ }
+
+ @Parameter(index = 3)
+ private int formatVersion;
+
+ private final String sourceTableName = "source_table";
+ private File fileTableDir;
+
+ @BeforeEach
+ public void setupTempDirs() {
+ fileTableDir = temp.toFile();
+ }
+
+ @AfterEach
+ public void dropTables() {
+ sql("DROP TABLE IF EXISTS %s PURGE", sourceTableName);
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @TestTemplate
+ public void addDataUnpartitioned() {
+ createUnpartitionedFileTable("parquet");
+
+ createIcebergTable("id Integer, name String, dept String, subdept String");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void deleteAndAddBackUnpartitioned() {
+ createUnpartitionedFileTable("parquet");
+
+ createIcebergTable("id Integer, name String, dept String, subdept String");
+
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ String deleteData = "DELETE FROM %s";
+ sql(deleteData, tableName);
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @Disabled // TODO Classpath issues prevent us from actually writing to a Spark ORC table
+ public void addDataUnpartitionedOrc() {
+ createUnpartitionedFileTable("orc");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result =
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`orc`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertThat(result).isEqualTo(2L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addAvroFile() throws Exception {
+ // Spark Session Catalog cannot load metadata tables
+ // with "The namespace in session catalog must have exactly one name part"
+ assumeThat(catalogName).isNotEqualTo("spark_catalog");
+
+ // Create an Avro file
+
+ Schema schema =
+ SchemaBuilder.record("record")
+ .fields()
+ .requiredInt("id")
+ .requiredString("data")
+ .endRecord();
+ GenericRecord record1 = new GenericData.Record(schema);
+ record1.put("id", 1L);
+ record1.put("data", "a");
+ GenericRecord record2 = new GenericData.Record(schema);
+ record2.put("id", 2L);
+ record2.put("data", "b");
+ File outputFile = temp.resolve("test.avro").toFile();
+
+ DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter(schema);
+ DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter(datumWriter);
+ dataFileWriter.create(schema, outputFile);
+ dataFileWriter.append(record1);
+ dataFileWriter.append(record2);
+ dataFileWriter.close();
+
+ createIcebergTable("id Long, data String");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`avro`.`%s`')",
+ catalogName, tableName, outputFile.getPath());
+ assertOutput(result, 1L, 1L);
+
+ List<Object[]> expected = Lists.newArrayList(new Object[] {1L, "a"}, new Object[] {2L, "b"});
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ expected,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+
+ List<Object[]> actualRecordCount =
+ sql("select %s from %s.files", DataFile.RECORD_COUNT.name(), tableName);
+ List<Object[]> expectedRecordCount = Lists.newArrayList();
+ expectedRecordCount.add(new Object[] {2L});
+ assertEquals(
+ "Iceberg file metadata should have correct metadata count",
+ expectedRecordCount,
+ actualRecordCount);
+ }
+
+ // TODO Adding spark-avro doesn't work in tests
+ @Disabled
+ public void addDataUnpartitionedAvro() {
+ createUnpartitionedFileTable("avro");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ Object result =
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`avro`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertThat(result).isEqualTo(2L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addDataUnpartitionedHive() {
+ createUnpartitionedHiveTable();
+
+ createIcebergTable("id Integer, name String, dept String, subdept String");
+
+ List<Object[]> result =
+ sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addDataUnpartitionedExtraCol() {
+ createUnpartitionedFileTable("parquet");
+
+ createIcebergTable("id Integer, name String, dept String, subdept String, foo string");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addDataUnpartitionedMissingCol() {
+ createUnpartitionedFileTable("parquet");
+
+ createIcebergTable("id Integer, name String, dept String");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addDataPartitionedMissingCol() {
+ createPartitionedFileTable("parquet");
+
+ createIcebergTable("id Integer, name String, dept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 8L, 4L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addDataPartitioned() {
+ createPartitionedFileTable("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 8L, 4L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @Disabled // TODO Classpath issues prevent us from actually writing to a Spark ORC table
+ public void addDataPartitionedOrc() {
+ createPartitionedFileTable("orc");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result =
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertThat(result).isEqualTo(8L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ // TODO Adding spark-avro doesn't work in tests
+ @Disabled
+ public void addDataPartitionedAvro() {
+ createPartitionedFileTable("avro");
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
+
+ sql(createIceberg, tableName);
+
+ Object result =
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`avro`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertThat(result).isEqualTo(8L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addDataPartitionedHive() {
+ createPartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result =
+ sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
+
+ assertOutput(result, 8L, 4L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addPartitionToPartitioned() {
+ createPartitionedFileTable("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void deleteAndAddBackPartitioned() {
+ createPartitionedFileTable("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ String deleteData = "DELETE FROM %s where id = 1";
+ sql(deleteData, tableName);
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addPartitionToPartitionedSnapshotIdInheritanceEnabledInTwoRuns()
+ throws NoSuchTableException, ParseException {
+ createPartitionedFileTable("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')",
+ tableName, TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED);
+
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 2))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id < 3 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+
+ manifestSpecMatchesTableSpec();
+
+ verifyUUIDInPath();
+ }
+
+ @TestTemplate
+ public void addPartitionsFromHiveSnapshotInheritanceEnabled()
+ throws NoSuchTableException, ParseException {
+ createPartitionedHiveTable();
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ sql(
+ "ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')",
+ tableName, TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED);
+
+ sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+
+ manifestSpecMatchesTableSpec();
+
+ verifyUUIDInPath();
+ }
+
+ @TestTemplate
+ public void addDataPartitionedByDateToPartitioned() {
+ createDatePartitionedFileTable("parquet");
+
+ createIcebergTable("id Integer, name String, date Date", "PARTITIONED BY (date)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('date', '2021-01-01'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, date FROM %s WHERE date = '2021-01-01' ORDER BY id", sourceTableName),
+ sql("SELECT id, name, date FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addDataPartitionedVerifyPartitionTypeInferredCorrectly() {
+ createTableWithTwoPartitions("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, date Date, dept String", "PARTITIONED BY (date, dept)");
+
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('date', '2021-01-01'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ String sqlFormat =
+ "SELECT id, name, dept, date FROM %s WHERE date = '2021-01-01' and dept= '01' ORDER BY id";
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql(sqlFormat, sourceTableName),
+ sql(sqlFormat, tableName));
+ }
+
+ @TestTemplate
+ public void addFilteredPartitionsToPartitioned() {
+ createCompositePartitionedTable("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addFilteredPartitionsToPartitioned2() {
+ createCompositePartitionedTable("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 6L, 3L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql(
+ "SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id",
+ sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnId() {
+ createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnDept() {
+ createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 6L, 3L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql(
+ "SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id",
+ sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addAllPartitionsToPartitionedWithNullValue() {
+ createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
+
+ // Add all partitions including null partitions.
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 10L, 5L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addAllPartitionsToNonStringPartitionedWithNullValue() {
+ createPartitionedTableWithNullValueInPartitionColumnOnId("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ // Add all partitions including null partitions.
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 10L, 5L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addPartitionsWithNullValueShouldAddFilesToNullPartition() {
+ // This test is to ensure that "null" string partition is not incorrectly created.
+
+ createPartitionedTableWithNullValueInPartitionColumnOnDept("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (dept)");
+
+ // Add all partitions including null partitions.
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 6L, 3L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+
+ // Check if correct partitions are created
+ List<Object[]> actualRows =
+ sql("SELECT partition from %s.partitions ORDER BY partition", tableName);
+ assertEquals(
+ "Other partitions should match",
+ ImmutableList.of(
+ row(new Object[] {new Object[] {null}}),
+ row(new Object[] {new Object[] {"facilities"}}),
+ row(new Object[] {new Object[] {"hr"}})),
+ actualRows);
+ }
+
+ @TestTemplate
+ public void addFileTableOldSpecDataAfterPartitionSpecEvolved()
+ throws NoSuchTableException, ParseException {
+ createPartitionedFileTable("parquet");
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String",
+ "PARTITIONED BY (id, dept, subdept)");
+ sql("ALTER TABLE %s DROP PARTITION FIELD dept", tableName);
+ sql(
+ "ALTER TABLE %s DROP PARTITION FIELD subdept",
+ tableName); // This spec now matches the partitioning of the parquet table
+ sql("ALTER TABLE %s ADD PARTITION FIELD subdept", tableName);
+
+ if (formatVersion == 1) {
+ // In V1, since we are dropping the partition field, it adds a void transform which will not
+ // match with the input spec
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Cannot find a partition spec in Iceberg table %s that matches the partition columns ([id]) in input table",
+ tableName);
+ return;
+ }
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files(table => '%s', source_table => '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+
+ assertOutput(result, 8L, 4L);
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+
+ Table table = Spark3Util.loadIcebergTable(spark, tableName);
+ // Find the spec that matches the partitioning of the parquet table
+ PartitionSpec compatibleSpec =
+ table.specs().values().stream()
+ .filter(spec -> spec.fields().size() == 1)
+ .filter(spec -> "id".equals(spec.fields().get(0).name()))
+ .findFirst()
+ .orElse(null);
+
+ assertThat(compatibleSpec).isNotNull();
+ manifestSpecMatchesGivenSpec(table, compatibleSpec);
+ verifyUUIDInPath();
+ }
+
+ @TestTemplate
+ public void addFileTableNoCompatibleSpec() {
+ createPartitionedFileTable("parquet");
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (dept)");
+ sql("ALTER TABLE %s ADD PARTITION FIELD subdept", tableName);
+
+ String fullTableName = tableName;
+ if (implementation.equals(SparkCatalogConfig.SPARK_SESSION.implementation())) {
+ fullTableName = String.format("%s.%s", catalogName, tableName);
+ }
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files(table => '%s', source_table => '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Cannot find a partition spec in Iceberg table %s that matches the partition columns (%s) in input table",
+ fullTableName, "[id]");
+ }
+
+ @TestTemplate
+ public void addWeirdCaseHiveTable() {
+ createWeirdCaseTable();
+
+ createIcebergTable(
+ "id Integer, `naMe` String, dept String, subdept String", "PARTITIONED BY (`naMe`)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '%s', map('naMe', 'John Doe'))",
+ catalogName, tableName, sourceTableName);
+
+ assertOutput(result, 2L, 1L);
+
+ /*
+ While we would like to use
+ SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id
+ Spark does not actually handle this pushdown correctly for hive based tables and it returns 0 records
+ */
+ List<Object[]> expected =
+ sql("SELECT id, `naMe`, dept, subdept from %s ORDER BY id", sourceTableName).stream()
+ .filter(r -> r[1].equals("John Doe"))
+ .collect(Collectors.toList());
+
+ // TODO when this assert breaks Spark fixed the pushdown issue
+ assertThat(
+ sql(
+ "SELECT id, `naMe`, dept, subdept from %s WHERE `naMe` = 'John Doe' ORDER BY id",
+ sourceTableName))
+ .as("If this assert breaks it means that Spark has fixed the pushdown issue")
+ .isEmpty();
+
+ // Pushdown works for iceberg
+ assertThat(
+ sql(
+ "SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id",
+ tableName))
+ .as("We should be able to pushdown mixed case partition keys")
+ .hasSize(2);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ expected,
+ sql("SELECT id, `naMe`, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void addPartitionToPartitionedHive() {
+ createPartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '%s', map('id', 1))",
+ catalogName, tableName, sourceTableName);
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void invalidDataImport() {
+ createPartitionedFileTable("parquet");
+
+ createIcebergTable("id Integer, name String, dept String, subdept String");
+
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
+ catalogName, tableName, fileTableDir.getAbsolutePath()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
+ .hasMessageContaining("that matches the partition columns");
+
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
+ .hasMessageContaining("that matches the partition columns");
+ }
+
+ @TestTemplate
+ public void invalidDataImportPartitioned() {
+ createUnpartitionedFileTable("parquet");
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('x', '1', 'y', '2'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
+ .hasMessageContaining("that matches the partition columns");
+
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', '2'))",
+ catalogName, tableName, fileTableDir.getAbsolutePath()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
+ .hasMessageContaining("that matches the partition columns");
+ }
+
+ @TestTemplate
+ public void partitionColumnCountMismatchInFilter() {
+ createPartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '%s', map('id', '0', 'dept', '1'))",
+ catalogName, tableName, sourceTableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith("Cannot add data files to target table")
+ .hasMessageContaining(
+ "because that table is partitioned, but the number of columns in the provided partition filter (2)"
+ + " is greater than the number of partitioned columns in table (1)");
+ }
+
+ @TestTemplate
+ public void invalidPartitionColumnsInFilter() {
+ createPartitionedHiveTable();
+
+ String icebergTablePartitionNames = "id";
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String",
+ String.format("PARTITIONED BY (%s)", icebergTablePartitionNames));
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '%s', map('dept', '1'))",
+ catalogName, tableName, sourceTableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith("Cannot add files to target table")
+ .hasMessageContaining(
+ "specified partition filter refers to columns that are not partitioned: [dept]")
+ .hasMessageContaining("Valid partition columns: [%s]", icebergTablePartitionNames);
+ }
+
+ @TestTemplate
+ public void addTwice() {
+ createPartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result1 =
+ sql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s', "
+ + "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName);
+ assertOutput(result1, 2L, 1L);
+
+ List<Object[]> result2 =
+ sql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s', "
+ + "partition_filter => map('id', 2))",
+ catalogName, tableName, sourceTableName);
+ assertOutput(result2, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", tableName));
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void duplicateDataPartitioned() {
+ createPartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ sql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s', "
+ + "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName);
+
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s', "
+ + "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageStartingWith(
+ "Cannot complete import because data files to be imported already"
+ + " exist within the target table");
+ }
+
+ @TestTemplate
+ public void duplicateDataPartitionedAllowed() {
+ createPartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result1 =
+ sql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s', "
+ + "partition_filter => map('id', 1))",
+ catalogName, tableName, sourceTableName);
+
+ assertOutput(result1, 2L, 1L);
+
+ List<Object[]> result2 =
+ sql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s', "
+ + "partition_filter => map('id', 1),"
+ + "check_duplicate_files => false)",
+ catalogName, tableName, sourceTableName);
+
+ assertOutput(result2, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql(
+ "SELECT id, name, dept, subdept FROM %s WHERE id = 1 UNION ALL "
+ + "SELECT id, name, dept, subdept FROM %s WHERE id = 1",
+ sourceTableName, sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s", tableName, tableName));
+ }
+
+ @TestTemplate
+ public void duplicateDataUnpartitioned() {
+ createUnpartitionedHiveTable();
+
+ createIcebergTable("id Integer, name String, dept String, subdept String");
+
+ sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
+
+ assertThatThrownBy(
+ () ->
+ scalarSql(
+ "CALL %s.system.add_files('%s', '%s')",
+ catalogName, tableName, sourceTableName))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageStartingWith(
+ "Cannot complete import because data files to be imported already"
+ + " exist within the target table");
+ }
+
+ @TestTemplate
+ public void duplicateDataUnpartitionedAllowed() {
+ createUnpartitionedHiveTable();
+
+ createIcebergTable("id Integer, name String, dept String, subdept String");
+
+ List<Object[]> result1 =
+ sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
+ assertOutput(result1, 2L, 1L);
+
+ List<Object[]> result2 =
+ sql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s',"
+ + "check_duplicate_files => false)",
+ catalogName, tableName, sourceTableName);
+ assertOutput(result2, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql(
+ "SELECT * FROM (SELECT * FROM %s UNION ALL " + "SELECT * from %s) ORDER BY id",
+ sourceTableName, sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void testEmptyImportDoesNotThrow() {
+ createIcebergTable("id Integer, name String, dept String, subdept String");
+
+ // Empty path based import
+ List<Object[]> pathResult =
+ sql(
+ "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
+ catalogName, tableName, fileTableDir.getAbsolutePath());
+ assertOutput(pathResult, 0L, 0L);
+ assertEquals(
+ "Iceberg table contains no added data when importing from an empty path",
+ EMPTY_QUERY_RESULT,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+
+ // Empty table based import
+ String createHive =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) STORED AS parquet";
+ sql(createHive, sourceTableName);
+
+ List<Object[]> tableResult =
+ sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
+ assertOutput(tableResult, 0L, 0L);
+ assertEquals(
+ "Iceberg table contains no added data when importing from an empty table",
+ EMPTY_QUERY_RESULT,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void testPartitionedImportFromEmptyPartitionDoesNotThrow() {
+ createPartitionedHiveTable();
+
+ final int emptyPartitionId = 999;
+ // Add an empty partition to the hive table
+ sql(
+ "ALTER TABLE %s ADD PARTITION (id = '%d') LOCATION '%d'",
+ sourceTableName, emptyPartitionId, emptyPartitionId);
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> tableResult =
+ sql(
+ "CALL %s.system.add_files("
+ + "table => '%s', "
+ + "source_table => '%s', "
+ + "partition_filter => map('id', %d))",
+ catalogName, tableName, sourceTableName, emptyPartitionId);
+
+ assertOutput(tableResult, 0L, 0L);
+ assertEquals(
+ "Iceberg table contains no added data when importing from an empty table",
+ EMPTY_QUERY_RESULT,
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void testAddFilesWithParallelism() {
+ createUnpartitionedHiveTable();
+
+ String createIceberg =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
+
+ sql(createIceberg, tableName);
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files(table => '%s', source_table => '%s', parallelism => 2)",
+ catalogName, tableName, sourceTableName);
+
+ assertEquals("Procedure output must match", ImmutableList.of(row(2L, 1L)), result);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT * FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT * FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void testAddFilesPartitionedWithParallelism() {
+ createPartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files(table => '%s', source_table => '%s', parallelism => 2)",
+ catalogName, tableName, sourceTableName);
+
+ assertOutput(result, 8L, 4L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ @TestTemplate
+ public void testAddFilesWithInvalidParallelism() {
+ createUnpartitionedHiveTable();
+
+ createIcebergTable(
+ "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
+
+ assertThatThrownBy(
+ () ->
+ sql(
+ "CALL %s.system.add_files(table => '%s', source_table => '%s', parallelism => -1)",
+ catalogName, tableName, sourceTableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Parallelism should be larger than 0");
+ }
+
+ @TestTemplate
+ public void testAddFilesToTableWithManySpecs() {
+ createPartitionedHiveTable();
+ createIcebergTable("id Integer, name String, dept String, subdept String"); // Spec 0
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD id", tableName); // Spec 1
+ sql("ALTER TABLE %s ADD PARTITION FIELD name", tableName); // Spec 2
+ sql("ALTER TABLE %s ADD PARTITION FIELD subdept", tableName); // Spec 3
+
+ List<Object[]> result =
+ sql(
+ "CALL %s.system.add_files('%s', '%s', map('id', 1))",
+ catalogName, tableName, sourceTableName);
+
+ assertOutput(result, 2L, 1L);
+
+ assertEquals(
+ "Iceberg table contains correct data",
+ sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
+ sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
+ }
+
+ private static final List<Object[]> EMPTY_QUERY_RESULT = Lists.newArrayList();
+
+ private static final StructField[] STRUCT = {
+ new StructField("id", DataTypes.IntegerType, true, Metadata.empty()),
+ new StructField("name", DataTypes.StringType, true, Metadata.empty()),
+ new StructField("dept", DataTypes.StringType, true, Metadata.empty()),
+ new StructField("subdept", DataTypes.StringType, true, Metadata.empty())
+ };
+
+ private Dataset<Row> unpartitionedDF() {
+ return spark
+ .createDataFrame(
+ ImmutableList.of(
+ RowFactory.create(1, "John Doe", "hr", "communications"),
+ RowFactory.create(2, "Jane Doe", "hr", "salary"),
+ RowFactory.create(3, "Matt Doe", "hr", "communications"),
+ RowFactory.create(4, "Will Doe", "facilities", "all")),
+ new StructType(STRUCT))
+ .repartition(1);
+ }
+
+ private Dataset<Row> singleNullRecordDF() {
+ return spark
+ .createDataFrame(
+ ImmutableList.of(RowFactory.create(null, null, null, null)), new StructType(STRUCT))
+ .repartition(1);
+ }
+
+ private Dataset<Row> partitionedDF() {
+ return unpartitionedDF().select("name", "dept", "subdept", "id");
+ }
+
+ private Dataset<Row> compositePartitionedDF() {
+ return unpartitionedDF().select("name", "subdept", "id", "dept");
+ }
+
+ private Dataset<Row> compositePartitionedNullRecordDF() {
+ return singleNullRecordDF().select("name", "subdept", "id", "dept");
+ }
+
+ private Dataset<Row> weirdColumnNamesDF() {
+ Dataset<Row> unpartitionedDF = unpartitionedDF();
+ return unpartitionedDF.select(
+ unpartitionedDF.col("id"),
+ unpartitionedDF.col("subdept"),
+ unpartitionedDF.col("dept"),
+ unpartitionedDF.col("name").as("naMe"));
+ }
+
+ private static final StructField[] DATE_STRUCT = {
+ new StructField("id", DataTypes.IntegerType, true, Metadata.empty()),
+ new StructField("name", DataTypes.StringType, true, Metadata.empty()),
+ new StructField("ts", DataTypes.DateType, true, Metadata.empty()),
+ new StructField("dept", DataTypes.StringType, true, Metadata.empty()),
+ };
+
+ private static java.sql.Date toDate(String value) {
+ return new java.sql.Date(DateTime.parse(value).getMillis());
+ }
+
+ private Dataset<Row> dateDF() {
+ return spark
+ .createDataFrame(
+ ImmutableList.of(
+ RowFactory.create(1, "John Doe", toDate("2021-01-01"), "01"),
+ RowFactory.create(2, "Jane Doe", toDate("2021-01-01"), "01"),
+ RowFactory.create(3, "Matt Doe", toDate("2021-01-02"), "02"),
+ RowFactory.create(4, "Will Doe", toDate("2021-01-02"), "02")),
+ new StructType(DATE_STRUCT))
+ .repartition(2);
+ }
+
+ private void createUnpartitionedFileTable(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s LOCATION '%s'";
+
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+ Dataset<Row> df = unpartitionedDF();
+ df.write().insertInto(sourceTableName);
+ df.write().insertInto(sourceTableName);
+ }
+
+ private void createPartitionedFileTable(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s PARTITIONED BY (id) "
+ + "LOCATION '%s'";
+
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ Dataset<Row> df = partitionedDF();
+ df.write().insertInto(sourceTableName);
+ df.write().insertInto(sourceTableName);
+ }
+
+ private void createCompositePartitionedTable(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s "
+ + "PARTITIONED BY (id, dept) LOCATION '%s'";
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ Dataset<Row> df = compositePartitionedDF();
+ df.write().insertInto(sourceTableName);
+ df.write().insertInto(sourceTableName);
+ }
+
+ private void createCompositePartitionedTableWithNullValueInPartitionColumn(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s "
+ + "PARTITIONED BY (id, dept) LOCATION '%s'";
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ Dataset<Row> unionedDF =
+ compositePartitionedDF()
+ .unionAll(compositePartitionedNullRecordDF())
+ .select("name", "subdept", "id", "dept")
+ .repartition(1);
+
+ unionedDF.write().insertInto(sourceTableName);
+ unionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createPartitionedTableWithNullValueInPartitionColumnOnDept(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s "
+ + "PARTITIONED BY (dept) LOCATION '%s'";
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ Dataset<Row> unionedDF =
+ unpartitionedDF()
+ .select("id", "name", "subdept", "dept")
+ .unionAll(singleNullRecordDF().select("id", "name", "subdept", "dept"))
+ .repartition(1);
+
+ unionedDF.write().insertInto(sourceTableName);
+ unionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createPartitionedTableWithNullValueInPartitionColumnOnId(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s "
+ + "PARTITIONED BY (id) LOCATION '%s'";
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ Dataset<Row> unionedDF =
+ unpartitionedDF()
+ .select("name", "subdept", "dept", "id")
+ .unionAll(singleNullRecordDF().select("name", "subdept", "dept", "id"))
+ .repartition(1);
+
+ unionedDF.write().insertInto(sourceTableName);
+ unionedDF.write().insertInto(sourceTableName);
+ }
+
+ private void createWeirdCaseTable() {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, subdept String, dept String) "
+ + "PARTITIONED BY (`naMe` String) STORED AS parquet";
+
+ sql(createParquet, sourceTableName);
+
+ Dataset<Row> df = weirdColumnNamesDF();
+ df.write().insertInto(sourceTableName);
+ df.write().insertInto(sourceTableName);
+ }
+
+ private void createUnpartitionedHiveTable() {
+ String createHive =
+ "CREATE TABLE %s (id Integer, name String, dept String, subdept String) STORED AS parquet";
+
+ sql(createHive, sourceTableName);
+
+ Dataset<Row> df = unpartitionedDF();
+ df.write().insertInto(sourceTableName);
+ df.write().insertInto(sourceTableName);
+ }
+
+ private void createPartitionedHiveTable() {
+ String createHive =
+ "CREATE TABLE %s (name String, dept String, subdept String) "
+ + "PARTITIONED BY (id Integer) STORED AS parquet";
+
+ sql(createHive, sourceTableName);
+
+ Dataset<Row> df = partitionedDF();
+ df.write().insertInto(sourceTableName);
+ df.write().insertInto(sourceTableName);
+ }
+
+ private void createDatePartitionedFileTable(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, date Date) USING %s "
+ + "PARTITIONED BY (date) LOCATION '%s'";
+
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ dateDF().select("id", "name", "ts").write().insertInto(sourceTableName);
+ }
+
+ private void createTableWithTwoPartitions(String format) {
+ String createParquet =
+ "CREATE TABLE %s (id Integer, name String, date Date, dept String) USING %s "
+ + "PARTITIONED BY (date, dept) LOCATION '%s'";
+
+ sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
+
+ dateDF().write().insertInto(sourceTableName);
+ }
+
+ private void createIcebergTable(String schema) {
+ createIcebergTable(schema, "");
+ }
+
+ private void createIcebergTable(String schema, String partitioning) {
+ sql(
+ "CREATE TABLE %s (%s) USING iceberg %s TBLPROPERTIES ('%s' '%d')",
+ tableName, schema, partitioning, TableProperties.FORMAT_VERSION, formatVersion);
+ }
+
+ private void assertOutput(
+ List<Object[]> result, long expectedAddedFilesCount, long expectedChangedPartitionCount) {
+ Object[] output = Iterables.getOnlyElement(result);
+ assertThat(output[0]).isEqualTo(expectedAddedFilesCount);
+ if (formatVersion == 1) {
+ assertThat(output[1]).isEqualTo(expectedChangedPartitionCount);
+ } else {
+ // the number of changed partitions may not be populated in v2 tables
+ assertThat(output[1]).isIn(expectedChangedPartitionCount, null);
+ }
+ }
+
+ private void manifestSpecMatchesTableSpec() throws NoSuchTableException, ParseException {
+ Table table = Spark3Util.loadIcebergTable(spark, tableName);
+ manifestSpecMatchesGivenSpec(table, table.spec());
+ }
+
+ private void manifestSpecMatchesGivenSpec(Table table, PartitionSpec partitionSpec) {
+ FileIO io = ((HasTableOperations) table).operations().io();
+ // Check that the manifests have the correct partition spec
+ assertThat(
+ table.currentSnapshot().allManifests(io).stream()
+ .map(mf -> ManifestFiles.read(mf, io, null /* force reading spec from file*/))
+ .map(ManifestReader::spec)
+ .collect(Collectors.toList()))
+ .allSatisfy(spec -> assertThat(spec).isEqualTo(partitionSpec));
+ }
+
+ private void verifyUUIDInPath() {
+ // verify manifest file name has uuid pattern
+ String manifestPath = (String) sql("select path from %s.manifests", tableName).get(0)[0];
+
+ Pattern uuidPattern = Pattern.compile("[a-f0-9]{8}(?:-[a-f0-9]{4}){4}[a-f0-9]{8}");
+
+ Matcher matcher = uuidPattern.matcher(manifestPath);
+ assertThat(matcher.find()).as("verify manifest path has uuid").isTrue();
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
new file mode 100644
index 0000000000..296564e20d
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -0,0 +1,650 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.List;
+import org.apache.iceberg.Parameter;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.SparkCatalogConfig;
+import org.apache.iceberg.spark.source.SparkTable;
+import org.apache.spark.sql.connector.catalog.CatalogManager;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestAlterTablePartitionFields extends ExtensionsTestBase {
+
+ @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}, formatVersion = {3}")
+ public static Object[][] parameters() {
+ return new Object[][] {
+ {
+ SparkCatalogConfig.HIVE.catalogName(),
+ SparkCatalogConfig.HIVE.implementation(),
+ SparkCatalogConfig.HIVE.properties(),
+ 1
+ },
+ {
+ SparkCatalogConfig.SPARK_SESSION.catalogName(),
+ SparkCatalogConfig.SPARK_SESSION.implementation(),
+ SparkCatalogConfig.SPARK_SESSION.properties(),
+ 2
+ }
+ };
+ }
+
+ @Parameter(index = 3)
+ private int formatVersion;
+
+ @AfterEach
+ public void removeTable() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @TestTemplate
+ public void testAddIdentityPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD category", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).identity("category").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddBucketPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .bucket("id", 16, "id_bucket_16")
+ .build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddTruncatePartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD truncate(data, 4)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .truncate("data", 4, "data_trunc_4")
+ .build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddYearsPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddMonthsPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD months(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddDaysPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddHoursPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hours(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddYearPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddMonthPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddDayPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddHourPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testAddNamedPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).bucket("id", 16, "shard").build();
+
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testDropIdentityPartition() {
+ createTable("id bigint NOT NULL, category string, data string", "category");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().fields()).as("Table should start with 1 partition field").hasSize(1);
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD category", tableName);
+
+ table.refresh();
+
+ if (formatVersion == 1) {
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .alwaysNull("category", "category")
+ .build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ } else {
+ assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
+ }
+ }
+
+ @TestTemplate
+ public void testDropDaysPartition() {
+ createTable("id bigint NOT NULL, ts timestamp, data string", "days(ts)");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().fields()).as("Table should start with 1 partition field").hasSize(1);
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD days(ts)", tableName);
+
+ table.refresh();
+
+ if (formatVersion == 1) {
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).alwaysNull("ts", "ts_day").build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ } else {
+ assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
+ }
+ }
+
+ @TestTemplate
+ public void testDropBucketPartition() {
+ createTable("id bigint NOT NULL, data string", "bucket(16, id)");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().fields()).as("Table should start with 1 partition field").hasSize(1);
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD bucket(16, id)", tableName);
+
+ table.refresh();
+
+ if (formatVersion == 1) {
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(1)
+ .alwaysNull("id", "id_bucket")
+ .build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ } else {
+ assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
+ }
+ }
+
+ @TestTemplate
+ public void testDropPartitionByName() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
+
+ table.refresh();
+
+ assertThat(table.spec().fields()).as("Table should have 1 partition field").hasSize(1);
+
+ // Should be recognized as iceberg command even with extra white spaces
+ sql("ALTER TABLE %s DROP PARTITION \n FIELD shard", tableName);
+
+ table.refresh();
+
+ if (formatVersion == 1) {
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(2).alwaysNull("id", "shard").build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+ } else {
+ assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
+ }
+ }
+
+ @TestTemplate
+ public void testReplacePartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
+ table.refresh();
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD days(ts) WITH hours(ts)", tableName);
+ table.refresh();
+ if (formatVersion == 1) {
+ expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "ts_day")
+ .hour("ts")
+ .build();
+ } else {
+ expected =
+ TestHelpers.newExpectedSpecBuilder()
+ .withSchema(table.schema())
+ .withSpecId(2)
+ .addField("hour", 3, 1001, "ts_hour")
+ .build();
+ }
+ assertThat(table.spec())
+ .as("Should changed from daily to hourly partitioned field")
+ .isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testReplacePartitionAndRename() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
+ table.refresh();
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD days(ts) WITH hours(ts) AS hour_col", tableName);
+ table.refresh();
+ if (formatVersion == 1) {
+ expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "ts_day")
+ .hour("ts", "hour_col")
+ .build();
+ } else {
+ expected =
+ TestHelpers.newExpectedSpecBuilder()
+ .withSchema(table.schema())
+ .withSpecId(2)
+ .addField("hour", 3, 1001, "hour_col")
+ .build();
+ }
+ assertThat(table.spec())
+ .as("Should changed from daily to hourly partitioned field")
+ .isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testReplaceNamedPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts) AS day_col", tableName);
+ table.refresh();
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts", "day_col").build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD day_col WITH hours(ts)", tableName);
+ table.refresh();
+ if (formatVersion == 1) {
+ expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "day_col")
+ .hour("ts")
+ .build();
+ } else {
+ expected =
+ TestHelpers.newExpectedSpecBuilder()
+ .withSchema(table.schema())
+ .withSpecId(2)
+ .addField("hour", 3, 1001, "ts_hour")
+ .build();
+ }
+ assertThat(table.spec())
+ .as("Should changed from daily to hourly partitioned field")
+ .isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testReplaceNamedPartitionAndRenameDifferently() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data string");
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD days(ts) AS day_col", tableName);
+ table.refresh();
+ PartitionSpec expected =
+ PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts", "day_col").build();
+ assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD day_col WITH hours(ts) AS hour_col", tableName);
+ table.refresh();
+ if (formatVersion == 1) {
+ expected =
+ PartitionSpec.builderFor(table.schema())
+ .withSpecId(2)
+ .alwaysNull("ts", "day_col")
+ .hour("ts", "hour_col")
+ .build();
+ } else {
+ expected =
+ TestHelpers.newExpectedSpecBuilder()
+ .withSchema(table.schema())
+ .withSpecId(2)
+ .addField("hour", 3, 1001, "hour_col")
+ .build();
+ }
+ assertThat(table.spec())
+ .as("Should changed from daily to hourly partitioned field")
+ .isEqualTo(expected);
+ }
+
+ @TestTemplate
+ public void testSparkTableAddDropPartitions() throws Exception {
+ createTable("id bigint NOT NULL, ts timestamp, data string");
+ assertThat(sparkTable().partitioning()).as("spark table partition should be empty").isEmpty();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
+ assertPartitioningEquals(sparkTable(), 1, "bucket(16, id)");
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD truncate(data, 4)", tableName);
+ assertPartitioningEquals(sparkTable(), 2, "truncate(4, data)");
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName);
+ assertPartitioningEquals(sparkTable(), 3, "years(ts)");
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD years(ts)", tableName);
+ assertPartitioningEquals(sparkTable(), 2, "truncate(4, data)");
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD truncate(4, data)", tableName);
+ assertPartitioningEquals(sparkTable(), 1, "bucket(16, id)");
+
+ sql("ALTER TABLE %s DROP PARTITION FIELD shard", tableName);
+ sql("DESCRIBE %s", tableName);
+ assertThat(sparkTable().partitioning()).as("spark table partition should be empty").isEmpty();
+ }
+
+ @TestTemplate
+ public void testDropColumnOfOldPartitionFieldV1() {
+ // default table created in v1 format
+ sql(
+ "CREATE TABLE %s (id bigint NOT NULL, ts timestamp, day_of_ts date) USING iceberg PARTITIONED BY (day_of_ts) TBLPROPERTIES('format-version' = '1')",
+ tableName);
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD day_of_ts WITH days(ts)", tableName);
+
+ sql("ALTER TABLE %s DROP COLUMN day_of_ts", tableName);
+ }
+
+ @TestTemplate
+ public void testDropColumnOfOldPartitionFieldV2() {
+ sql(
+ "CREATE TABLE %s (id bigint NOT NULL, ts timestamp, day_of_ts date) USING iceberg PARTITIONED BY (day_of_ts) TBLPROPERTIES('format-version' = '2')",
+ tableName);
+
+ sql("ALTER TABLE %s REPLACE PARTITION FIELD day_of_ts WITH days(ts)", tableName);
+
+ sql("ALTER TABLE %s DROP COLUMN day_of_ts", tableName);
+ }
+
+ private void assertPartitioningEquals(SparkTable table, int len, String transform) {
+ assertThat(table.partitioning()).as("spark table partition should be " + len).hasSize(len);
+ assertThat(table.partitioning()[len - 1])
+ .asString()
+ .as("latest spark table partition transform should match")
+ .isEqualTo(transform);
+ }
+
+ private SparkTable sparkTable() throws Exception {
+ validationCatalog.loadTable(tableIdent).refresh();
+ CatalogManager catalogManager = spark.sessionState().catalogManager();
+ TableCatalog catalog = (TableCatalog) catalogManager.catalog(catalogName);
+ Identifier identifier = Identifier.of(tableIdent.namespace().levels(), tableIdent.name());
+ return (SparkTable) catalog.loadTable(identifier);
+ }
+
+ private void createTable(String schema) {
+ createTable(schema, null);
+ }
+
+ private void createTable(String schema, String spec) {
+ if (spec == null) {
+ sql(
+ "CREATE TABLE %s (%s) USING iceberg TBLPROPERTIES ('%s' '%d')",
+ tableName, schema, TableProperties.FORMAT_VERSION, formatVersion);
+ } else {
+ sql(
+ "CREATE TABLE %s (%s) USING iceberg PARTITIONED BY (%s) TBLPROPERTIES ('%s' '%d')",
+ tableName, schema, spec, TableProperties.FORMAT_VERSION, formatVersion);
+ }
+ }
+
+ private void runCreateAndDropPartitionField(
+ String column, String partitionType, List<Object[]> expected, String predicate) {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ sql(
+ "CREATE TABLE %s (col_int INTEGER, col_ts TIMESTAMP_NTZ, col_long BIGINT) USING ICEBERG TBLPROPERTIES ('format-version' = %d)",
+ tableName, formatVersion);
+ sql("INSERT INTO %s VALUES (1000, CAST('2024-03-01 19:25:00' as TIMESTAMP), 2100)", tableName);
+ sql("ALTER TABLE %s ADD PARTITION FIELD %s AS col2_partition", tableName, partitionType);
+ sql("INSERT INTO %s VALUES (2000, CAST('2024-04-01 19:25:00' as TIMESTAMP), 2200)", tableName);
+ sql("ALTER TABLE %s DROP PARTITION FIELD col2_partition", tableName);
+ sql("INSERT INTO %s VALUES (3000, CAST('2024-05-01 19:25:00' as TIMESTAMP), 2300)", tableName);
+ sql("ALTER TABLE %s DROP COLUMN %s", tableName, column);
+
+ assertThat(sql("SELECT * FROM %s WHERE %s ORDER BY col_int", tableName, predicate))
+ .containsExactlyElementsOf(expected);
+ }
+
+ @TestTemplate
+ public void testDropPartitionAndSourceColumnLong() {
+ String predicateTs = "col_long >= 2200";
+ List<Object[]> expectedTs =
+ Lists.newArrayList(new Object[] {2000, 2200L}, new Object[] {3000, 2300L});
+ runCreateAndDropPartitionField("col_ts", "col_ts", expectedTs, predicateTs);
+ runCreateAndDropPartitionField("col_ts", "year(col_ts)", expectedTs, predicateTs);
+ runCreateAndDropPartitionField("col_ts", "month(col_ts)", expectedTs, predicateTs);
+ runCreateAndDropPartitionField("col_ts", "day(col_ts)", expectedTs, predicateTs);
+ }
+
+ @TestTemplate
+ public void testDropPartitionAndSourceColumnTimestamp() {
+ String predicate = "col_ts >= '2024-04-01 19:25:00'";
+ List<Object[]> expected =
+ Lists.newArrayList(
+ new Object[] {2000, LocalDateTime.ofEpochSecond(1711999500, 0, ZoneOffset.UTC)},
+ new Object[] {3000, LocalDateTime.ofEpochSecond(1714591500, 0, ZoneOffset.UTC)});
+ runCreateAndDropPartitionField("col_long", "col_long", expected, predicate);
+ runCreateAndDropPartitionField("col_long", "truncate(2, col_long)", expected, predicate);
+ runCreateAndDropPartitionField("col_long", "bucket(16, col_long)", expected, predicate);
+ }
+
+ @TestTemplate
+ public void deleteAfterDroppingPartitionAndSourceColumn() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ sql(
+ "CREATE TABLE %s (id INTEGER, data STRING) USING ICEBERG TBLPROPERTIES ('format-version' = %d)",
+ tableName, formatVersion);
+ sql("INSERT INTO %s VALUES (1, 'data1')", tableName);
+ sql("ALTER TABLE %s ADD PARTITION FIELD data", tableName);
+ sql("INSERT INTO %s VALUES (2, 'data2')", tableName);
+ sql("ALTER TABLE %s DROP PARTITION FIELD data", tableName);
+ sql("INSERT INTO %s VALUES (3, 'data3')", tableName);
+ sql("ALTER TABLE %s DROP COLUMN data", tableName);
+
+ assertThat(sql("SELECT * FROM %s WHERE id >= 1 ORDER BY id", tableName))
+ .containsExactly(row(1), row(2), row(3));
+
+ sql("DELETE FROM %s WHERE id >= 1", tableName);
+ assertThat(sql("SELECT * FROM %s WHERE id >= 1", tableName)).isEmpty();
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
new file mode 100644
index 0000000000..f36a2e4470
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Table;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestAlterTableSchema extends ExtensionsTestBase {
+
+ @AfterEach
+ public void removeTable() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @TestTemplate
+ public void testSetIdentifierFields() {
+ sql(
+ "CREATE TABLE %s (id bigint NOT NULL, "
+ + "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg",
+ tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.schema().identifierFieldIds())
+ .as("Table should start without identifier")
+ .isEmpty();
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id", tableName);
+ table.refresh();
+ assertThat(table.schema().identifierFieldIds())
+ .containsExactly(table.schema().findField("id").fieldId());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ assertThat(table.schema().identifierFieldIds())
+ .as("Should have new identifier field")
+ .containsExactlyInAnyOrder(
+ table.schema().findField("id").fieldId(),
+ table.schema().findField("location.lon").fieldId());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS location.lon", tableName);
+ table.refresh();
+ assertThat(table.schema().identifierFieldIds())
+ .as("Should have new identifier field")
+ .containsExactly(table.schema().findField("location.lon").fieldId());
+ }
+
+ @TestTemplate
+ public void testSetInvalidIdentifierFields() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, id2 bigint) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.schema().identifierFieldIds())
+ .as("Table should start without identifier")
+ .isEmpty();
+ assertThatThrownBy(() -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS unknown", tableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageEndingWith("not found in current schema or added columns");
+
+ assertThatThrownBy(() -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS id2", tableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageEndingWith("not a required field");
+ }
+
+ @TestTemplate
+ public void testDropIdentifierFields() {
+ sql(
+ "CREATE TABLE %s (id bigint NOT NULL, "
+ + "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg",
+ tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.schema().identifierFieldIds())
+ .as("Table should start without identifier")
+ .isEmpty();
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ assertThat(table.schema().identifierFieldIds())
+ .as("Should have new identifier fields")
+ .containsExactlyInAnyOrder(
+ table.schema().findField("id").fieldId(),
+ table.schema().findField("location.lon").fieldId());
+
+ sql("ALTER TABLE %s DROP IDENTIFIER FIELDS id", tableName);
+ table.refresh();
+ assertThat(table.schema().identifierFieldIds())
+ .as("Should removed identifier field")
+ .containsExactly(table.schema().findField("location.lon").fieldId());
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ assertThat(table.schema().identifierFieldIds())
+ .as("Should have new identifier fields")
+ .containsExactlyInAnyOrder(
+ table.schema().findField("id").fieldId(),
+ table.schema().findField("location.lon").fieldId());
+
+ sql("ALTER TABLE %s DROP IDENTIFIER FIELDS id, location.lon", tableName);
+ table.refresh();
+ assertThat(table.schema().identifierFieldIds()).as("Should have no identifier field").isEmpty();
+ }
+
+ @TestTemplate
+ public void testDropInvalidIdentifierFields() {
+ sql(
+ "CREATE TABLE %s (id bigint NOT NULL, data string NOT NULL, "
+ + "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg",
+ tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ assertThat(table.schema().identifierFieldIds())
+ .as("Table should start without identifier")
+ .isEmpty();
+ assertThatThrownBy(() -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS unknown", tableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot complete drop identifier fields operation: field unknown not found");
+
+ sql("ALTER TABLE %s SET IDENTIFIER FIELDS id", tableName);
+ assertThatThrownBy(() -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS data", tableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot complete drop identifier fields operation: data is not an identifier field");
+
+ assertThatThrownBy(() -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS location.lon", tableName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot complete drop identifier fields operation: location.lon is not an identifier field");
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java
new file mode 100644
index 0000000000..2408f3cfa2
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.List;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.spark.sql.AnalysisException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestAncestorsOfProcedure extends ExtensionsTestBase {
+
+ @AfterEach
+ public void removeTables() {
+ sql("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+ @TestTemplate
+ public void testAncestorOfUsingEmptyArgs() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'b')", tableName);
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Long currentSnapshotId = table.currentSnapshot().snapshotId();
+ Long currentTimestamp = table.currentSnapshot().timestampMillis();
+ Long preSnapshotId = table.currentSnapshot().parentId();
+ Long preTimeStamp = table.snapshot(table.currentSnapshot().parentId()).timestampMillis();
+
+ List<Object[]> output = sql("CALL %s.system.ancestors_of('%s')", catalogName, tableIdent);
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(currentSnapshotId, currentTimestamp), row(preSnapshotId, preTimeStamp)),
+ output);
+ }
+
+ @TestTemplate
+ public void testAncestorOfUsingSnapshotId() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'b')", tableName);
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Long currentSnapshotId = table.currentSnapshot().snapshotId();
+ Long currentTimestamp = table.currentSnapshot().timestampMillis();
+ Long preSnapshotId = table.currentSnapshot().parentId();
+ Long preTimeStamp = table.snapshot(table.currentSnapshot().parentId()).timestampMillis();
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(currentSnapshotId, currentTimestamp), row(preSnapshotId, preTimeStamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, currentSnapshotId));
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(row(preSnapshotId, preTimeStamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, preSnapshotId));
+ }
+
+ @TestTemplate
+ public void testAncestorOfWithRollBack() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ Table table = validationCatalog.loadTable(tableIdent);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+ table.refresh();
+ Long firstSnapshotId = table.currentSnapshot().snapshotId();
+ Long firstTimestamp = table.currentSnapshot().timestampMillis();
+ sql("INSERT INTO TABLE %s VALUES (2, 'b')", tableName);
+ table.refresh();
+ Long secondSnapshotId = table.currentSnapshot().snapshotId();
+ Long secondTimestamp = table.currentSnapshot().timestampMillis();
+ sql("INSERT INTO TABLE %s VALUES (3, 'c')", tableName);
+ table.refresh();
+ Long thirdSnapshotId = table.currentSnapshot().snapshotId();
+ Long thirdTimestamp = table.currentSnapshot().timestampMillis();
+
+ // roll back
+ sql(
+ "CALL %s.system.rollback_to_snapshot('%s', %dL)",
+ catalogName, tableIdent, secondSnapshotId);
+
+ sql("INSERT INTO TABLE %s VALUES (4, 'd')", tableName);
+ table.refresh();
+ Long fourthSnapshotId = table.currentSnapshot().snapshotId();
+ Long fourthTimestamp = table.currentSnapshot().timestampMillis();
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(fourthSnapshotId, fourthTimestamp),
+ row(secondSnapshotId, secondTimestamp),
+ row(firstSnapshotId, firstTimestamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, fourthSnapshotId));
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(
+ row(thirdSnapshotId, thirdTimestamp),
+ row(secondSnapshotId, secondTimestamp),
+ row(firstSnapshotId, firstTimestamp)),
+ sql("CALL %s.system.ancestors_of('%s', %dL)", catalogName, tableIdent, thirdSnapshotId));
+ }
+
+ @TestTemplate
+ public void testAncestorOfUsingNamedArgs() {
+ sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
+
+ Table table = validationCatalog.loadTable(tableIdent);
+ Long firstSnapshotId = table.currentSnapshot().snapshotId();
+ Long firstTimestamp = table.currentSnapshot().timestampMillis();
+
+ assertEquals(
+ "Procedure output must match",
+ ImmutableList.of(row(firstSnapshotId, firstTimestamp)),
+ sql(
+ "CALL %s.system.ancestors_of(snapshot_id => %dL, table => '%s')",
+ catalogName, firstSnapshotId, tableIdent));
+ }
+
+ @TestTemplate
+ public void testInvalidAncestorOfCases() {
+ assertThatThrownBy(() -> sql("CALL %s.system.ancestors_of()", catalogName))
+ .isInstanceOf(AnalysisException.class)
+ .hasMessage(
+ "[REQUIRED_PARAMETER_NOT_FOUND] Cannot invoke routine `ancestors_of` because the parameter named `table` is required, but the routine call did not supply a value. Please update the routine call to supply an argument value (either positionally at index 0 or by name) and retry the query again. SQLSTATE: 4274K");
+
+ assertThatThrownBy(() -> sql("CALL %s.system.ancestors_of('')", catalogName))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot handle an empty identifier for parameter 'table'");
+
+ assertThatThrownBy(
+ () -> sql("CALL %s.system.ancestors_of('%s', '1.1')", catalogName, tableIdent))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith(
+ "[CAST_INVALID_INPUT] The value '1.1' of the type \"STRING\" cannot be cast to \"BIGINT\" because it is malformed.");
+ }
+}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
new file mode 100644
index 0000000000..c49def204f
--- /dev/null
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.spark.SparkCatalogConfig;
+import org.apache.iceberg.spark.source.SimpleRecord;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.parser.extensions.IcebergParseException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestBranchDDL extends ExtensionsTestBase {
+
... 134402 lines suppressed ...