You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2025/05/08 06:24:39 UTC
(iceberg) branch revert-12494-spark4.0 created (now 21d145922f)
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a change to branch revert-12494-spark4.0
in repository https://gitbox.apache.org/repos/asf/iceberg.git
at 21d145922f Revert "Spark: Spark 4.0 initial support (#12494)"
This branch includes the following new commits:
new 21d145922f Revert "Spark: Spark 4.0 initial support (#12494)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iceberg) 01/01: Revert "Spark: Spark 4.0 initial support (#12494)"
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch revert-12494-spark4.0
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 21d145922f5beaee448032fda1c13c5eb14bddca
Author: Huaxin Gao <hu...@gmail.com>
AuthorDate: Wed May 7 23:24:33 2025 -0700
Revert "Spark: Spark 4.0 initial support (#12494)"
This reverts commit ad7f5c439b8392bd13d6376050e21330c2dfeced.
---
.github/workflows/java-ci.yml | 4 +-
.github/workflows/jmh-benchmarks.yml | 4 +-
.github/workflows/publish-snapshot.yml | 2 +-
.github/workflows/recurring-jmh-benchmarks.yml | 2 +-
.github/workflows/spark-ci.yml | 6 +-
.gitignore | 2 -
build.gradle | 3 -
.../java/org/apache/iceberg/MetadataColumns.java | 2 +-
gradle.properties | 4 +-
gradle/libs.versions.toml | 4 -
.../org/apache/iceberg/hive/TestHiveMetastore.java | 1 -
jmh.gradle | 5 -
settings.gradle | 12 -
spark/build.gradle | 4 -
spark/v4.0/build.gradle | 337 ---
.../apache/iceberg/DeleteFileIndexBenchmark.java | 279 --
.../spark/MergeCardinalityCheckBenchmark.java | 231 --
.../apache/iceberg/spark/PlanningBenchmark.java | 409 ---
.../iceberg/spark/TaskGroupPlanningBenchmark.java | 273 --
.../iceberg/spark/UpdateProjectionBenchmark.java | 213 --
.../IcebergSqlExtensions.g4 | 374 ---
.../extensions/IcebergSparkSessionExtensions.scala | 49 -
.../spark/sql/catalyst/analysis/CheckViews.scala | 117 -
.../analysis/ProcedureArgumentCoercion.scala | 55 -
.../sql/catalyst/analysis/ResolveProcedures.scala | 189 --
.../spark/sql/catalyst/analysis/ResolveViews.scala | 173 --
.../catalyst/analysis/RewriteViewCommands.scala | 181 --
.../spark/sql/catalyst/analysis/ViewUtil.scala | 51 -
.../catalyst/optimizer/ReplaceStaticInvoke.scala | 119 -
.../IcebergSparkSqlExtensionsParser.scala | 352 ---
.../IcebergSqlExtensionsAstBuilder.scala | 387 ---
.../catalyst/plans/logical/AddPartitionField.scala | 33 -
.../sql/catalyst/plans/logical/BranchOptions.scala | 23 -
.../plans/logical/CreateOrReplaceBranch.scala | 39 -
.../plans/logical/CreateOrReplaceTag.scala | 39 -
.../sql/catalyst/plans/logical/DropBranch.scala | 33 -
.../plans/logical/DropIdentifierFields.scala | 34 -
.../plans/logical/DropPartitionField.scala | 33 -
.../spark/sql/catalyst/plans/logical/DropTag.scala | 33 -
.../sql/catalyst/plans/logical/IcebergCall.scala | 34 -
.../plans/logical/ReplacePartitionField.scala | 38 -
.../plans/logical/SetIdentifierFields.scala | 35 -
.../sql/catalyst/plans/logical/TagOptions.scala | 22 -
.../sql/catalyst/plans/logical/statements.scala | 44 -
.../plans/logical/views/CreateIcebergView.scala | 44 -
.../plans/logical/views/DropIcebergView.scala | 30 -
.../plans/logical/views/ResolvedV2View.scala | 31 -
.../plans/logical/views/ShowIcebergViews.scala | 36 -
.../datasources/v2/AddPartitionFieldExec.scala | 56 -
.../v2/AlterV2ViewSetPropertiesExec.scala | 49 -
.../v2/AlterV2ViewUnsetPropertiesExec.scala | 55 -
.../sql/execution/datasources/v2/CallExec.scala | 40 -
.../datasources/v2/CreateOrReplaceBranchExec.scala | 100 -
.../datasources/v2/CreateOrReplaceTagExec.scala | 84 -
.../datasources/v2/CreateV2ViewExec.scala | 127 -
.../datasources/v2/DescribeV2ViewExec.scala | 79 -
.../execution/datasources/v2/DropBranchExec.scala | 56 -
.../datasources/v2/DropIdentifierFieldsExec.scala | 65 -
.../datasources/v2/DropPartitionFieldExec.scala | 67 -
.../sql/execution/datasources/v2/DropTagExec.scala | 56 -
.../execution/datasources/v2/DropV2ViewExec.scala | 48 -
.../v2/ExtendedDataSourceV2Strategy.scala | 175 --
.../datasources/v2/RenameV2ViewExec.scala | 45 -
.../datasources/v2/ReplacePartitionFieldExec.scala | 72 -
.../datasources/v2/SetIdentifierFieldsExec.scala | 53 -
.../v2/SetWriteDistributionAndOrderingExec.scala | 81 -
.../datasources/v2/ShowCreateV2ViewExec.scala | 79 -
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 55 -
.../execution/datasources/v2/ShowV2ViewsExec.scala | 67 -
.../apache/iceberg/spark/extensions/Employee.java | 66 -
.../spark/extensions/ExtensionsTestBase.java | 70 -
.../iceberg/spark/extensions/ProcedureUtil.java | 54 -
.../iceberg/spark/extensions/SparkPlanUtil.java | 107 -
.../SparkRowLevelOperationsTestBase.java | 461 ---
.../spark/extensions/TestAddFilesProcedure.java | 1373 ---------
.../extensions/TestAlterTablePartitionFields.java | 586 ----
.../spark/extensions/TestAlterTableSchema.java | 148 -
.../spark/extensions/TestAncestorsOfProcedure.java | 160 --
.../iceberg/spark/extensions/TestBranchDDL.java | 432 ---
.../spark/extensions/TestCallStatementParser.java | 219 --
.../spark/extensions/TestChangelogTable.java | 453 ---
.../TestCherrypickSnapshotProcedure.java | 189 --
.../extensions/TestComputeTableStatsProcedure.java | 137 -
.../spark/extensions/TestConflictValidation.java | 394 ---
.../spark/extensions/TestCopyOnWriteDelete.java | 227 --
.../spark/extensions/TestCopyOnWriteMerge.java | 195 --
.../spark/extensions/TestCopyOnWriteUpdate.java | 183 --
.../TestCreateChangelogViewProcedure.java | 694 -----
.../iceberg/spark/extensions/TestDelete.java | 1486 ----------
.../extensions/TestExpireSnapshotsProcedure.java | 559 ----
.../extensions/TestFastForwardBranchProcedure.java | 257 --
.../apache/iceberg/spark/extensions/TestMerge.java | 3016 -------------------
.../spark/extensions/TestMergeOnReadDelete.java | 352 ---
.../spark/extensions/TestMergeOnReadMerge.java | 173 --
.../spark/extensions/TestMergeOnReadUpdate.java | 226 --
.../TestMetaColumnProjectionWithStageScan.java | 125 -
.../spark/extensions/TestMetadataTables.java | 993 -------
.../extensions/TestMigrateTableProcedure.java | 289 --
.../extensions/TestPublishChangesProcedure.java | 183 --
.../extensions/TestRegisterTableProcedure.java | 85 -
.../extensions/TestRemoveOrphanFilesProcedure.java | 747 -----
.../spark/extensions/TestReplaceBranch.java | 270 --
.../TestRequiredDistributionAndOrdering.java | 313 --
.../extensions/TestRewriteDataFilesProcedure.java | 988 -------
.../extensions/TestRewriteManifestsProcedure.java | 384 ---
.../extensions/TestRewritePositionDeleteFiles.java | 418 ---
.../TestRewritePositionDeleteFilesProcedure.java | 250 --
.../extensions/TestRewriteTablePathProcedure.java | 176 --
.../TestRollbackToSnapshotProcedure.java | 279 --
.../TestRollbackToTimestampProcedure.java | 331 ---
.../TestSetCurrentSnapshotProcedure.java | 286 --
.../TestSetWriteDistributionAndOrdering.java | 339 ---
.../extensions/TestSnapshotTableProcedure.java | 283 --
...toragePartitionedJoinsInRowLevelOperations.java | 316 --
.../extensions/TestSystemFunctionPushDownDQL.java | 313 --
...SystemFunctionPushDownInRowLevelOperations.java | 365 ---
.../iceberg/spark/extensions/TestTagDDL.java | 365 ---
.../iceberg/spark/extensions/TestUpdate.java | 1518 ----------
.../apache/iceberg/spark/extensions/TestViews.java | 2125 --------------
.../iceberg/spark/extensions/TestWriteAborts.java | 185 --
.../org/apache/iceberg/spark/source/PlanUtils.java | 99 -
spark/v4.0/spark-runtime/LICENSE | 538 ----
spark/v4.0/spark-runtime/NOTICE | 403 ---
.../java/org/apache/iceberg/spark/SmokeTest.java | 182 --
.../apache/iceberg/spark/SparkBenchmarkUtil.java | 58 -
.../spark/action/DeleteOrphanFilesBenchmark.java | 189 --
.../action/IcebergSortCompactionBenchmark.java | 404 ---
.../iceberg/spark/action/RandomGeneratingUDF.java | 50 -
.../SparkParquetReadersFlatDataBenchmark.java | 229 --
.../SparkParquetReadersNestedDataBenchmark.java | 227 --
.../SparkParquetWritersFlatDataBenchmark.java | 130 -
.../SparkParquetWritersNestedDataBenchmark.java | 130 -
.../org/apache/iceberg/spark/source/Action.java | 24 -
.../iceberg/spark/source/DVReaderBenchmark.java | 267 --
.../iceberg/spark/source/DVWriterBenchmark.java | 251 --
.../spark/source/IcebergSourceBenchmark.java | 209 --
.../spark/source/IcebergSourceDeleteBenchmark.java | 340 ---
.../source/IcebergSourceFlatDataBenchmark.java | 59 -
.../source/IcebergSourceNestedDataBenchmark.java | 59 -
.../IcebergSourceNestedListDataBenchmark.java | 62 -
.../iceberg/spark/source/WritersBenchmark.java | 393 ---
.../spark/source/avro/AvroWritersBenchmark.java | 39 -
.../IcebergSourceFlatAvroDataReadBenchmark.java | 142 -
.../IcebergSourceNestedAvroDataReadBenchmark.java | 142 -
.../orc/IcebergSourceFlatORCDataBenchmark.java | 68 -
.../orc/IcebergSourceFlatORCDataReadBenchmark.java | 210 --
...ebergSourceNestedListORCDataWriteBenchmark.java | 109 -
.../IcebergSourceNestedORCDataReadBenchmark.java | 183 --
...cebergSourceFlatParquetDataFilterBenchmark.java | 129 -
.../IcebergSourceFlatParquetDataReadBenchmark.java | 165 --
...IcebergSourceFlatParquetDataWriteBenchmark.java | 89 -
...gSourceNestedListParquetDataWriteBenchmark.java | 89 -
...bergSourceNestedParquetDataFilterBenchmark.java | 128 -
...cebergSourceNestedParquetDataReadBenchmark.java | 166 --
...ebergSourceNestedParquetDataWriteBenchmark.java | 88 -
.../IcebergSourceParquetEqDeleteBenchmark.java | 57 -
...ebergSourceParquetMultiDeleteFileBenchmark.java | 57 -
.../IcebergSourceParquetPosDeleteBenchmark.java | 60 -
...gSourceParquetWithUnrelatedDeleteBenchmark.java | 64 -
.../source/parquet/ParquetWritersBenchmark.java | 39 -
...dDictionaryEncodedFlatParquetDataBenchmark.java | 127 -
.../VectorizedReadFlatParquetDataBenchmark.java | 355 ---
.../VectorizedReadParquetDecimalBenchmark.java | 207 --
.../apache/iceberg/SparkDistributedDataScan.java | 272 --
.../java/org/apache/iceberg/spark/BaseCatalog.java | 95 -
.../iceberg/spark/BaseFileRewriteCoordinator.java | 81 -
.../apache/iceberg/spark/ChangelogIterator.java | 127 -
.../org/apache/iceberg/spark/CommitMetadata.java | 66 -
.../iceberg/spark/ComputeUpdateIterator.java | 140 -
.../org/apache/iceberg/spark/ExtendedParser.java | 70 -
.../iceberg/spark/FileRewriteCoordinator.java | 32 -
.../org/apache/iceberg/spark/IcebergSpark.java | 56 -
.../org/apache/iceberg/spark/JobGroupInfo.java | 48 -
.../org/apache/iceberg/spark/JobGroupUtils.java | 64 -
.../org/apache/iceberg/spark/OrcBatchReadConf.java | 27 -
.../apache/iceberg/spark/ParquetBatchReadConf.java | 29 -
.../apache/iceberg/spark/ParquetReaderType.java | 47 -
.../org/apache/iceberg/spark/PathIdentifier.java | 57 -
.../spark/PositionDeletesRewriteCoordinator.java | 33 -
.../spark/PruneColumnsWithoutReordering.java | 242 --
.../iceberg/spark/RemoveCarryoverIterator.java | 148 -
.../iceberg/spark/RemoveNetCarryoverIterator.java | 129 -
.../apache/iceberg/spark/RollbackStagedTable.java | 142 -
.../apache/iceberg/spark/ScanTaskSetManager.java | 73 -
.../org/apache/iceberg/spark/SortOrderToSpark.java | 101 -
.../java/org/apache/iceberg/spark/Spark3Util.java | 1066 -------
.../org/apache/iceberg/spark/SparkAggregates.java | 84 -
.../iceberg/spark/SparkCachedTableCatalog.java | 211 --
.../org/apache/iceberg/spark/SparkCatalog.java | 1018 -------
.../apache/iceberg/spark/SparkCompressionUtil.java | 148 -
.../org/apache/iceberg/spark/SparkConfParser.java | 320 ---
.../org/apache/iceberg/spark/SparkContentFile.java | 277 --
.../org/apache/iceberg/spark/SparkDataFile.java | 46 -
.../org/apache/iceberg/spark/SparkDeleteFile.java | 40 -
.../apache/iceberg/spark/SparkExceptionUtil.java | 64 -
.../apache/iceberg/spark/SparkExecutorCache.java | 228 --
.../org/apache/iceberg/spark/SparkFilters.java | 280 --
.../org/apache/iceberg/spark/SparkFixupTypes.java | 63 -
.../apache/iceberg/spark/SparkFunctionCatalog.java | 45 -
.../org/apache/iceberg/spark/SparkReadConf.java | 366 ---
.../org/apache/iceberg/spark/SparkReadOptions.java | 98 -
.../apache/iceberg/spark/SparkSQLProperties.java | 104 -
.../org/apache/iceberg/spark/SparkSchemaUtil.java | 368 ---
.../apache/iceberg/spark/SparkSessionCatalog.java | 526 ----
.../org/apache/iceberg/spark/SparkStructLike.java | 54 -
.../org/apache/iceberg/spark/SparkTableCache.java | 54 -
.../org/apache/iceberg/spark/SparkTableUtil.java | 1261 --------
.../org/apache/iceberg/spark/SparkTypeToType.java | 162 --
.../org/apache/iceberg/spark/SparkTypeVisitor.java | 78 -
.../java/org/apache/iceberg/spark/SparkUtil.java | 339 ---
.../org/apache/iceberg/spark/SparkV2Filters.java | 484 ----
.../apache/iceberg/spark/SparkValueConverter.java | 157 -
.../org/apache/iceberg/spark/SparkWriteConf.java | 728 -----
.../apache/iceberg/spark/SparkWriteOptions.java | 88 -
.../iceberg/spark/SparkWriteRequirements.java | 59 -
.../org/apache/iceberg/spark/SparkWriteUtil.java | 259 --
.../apache/iceberg/spark/SupportsFunctions.java | 63 -
.../apache/iceberg/spark/SupportsReplaceView.java | 56 -
.../org/apache/iceberg/spark/TypeToSparkType.java | 140 -
.../actions/BaseSnapshotUpdateSparkAction.java | 47 -
.../iceberg/spark/actions/BaseSparkAction.java | 443 ---
.../actions/BaseTableCreationSparkAction.java | 182 --
.../actions/ComputeTableStatsSparkAction.java | 179 --
.../actions/DeleteOrphanFilesSparkAction.java | 676 -----
.../actions/DeleteReachableFilesSparkAction.java | 156 -
.../spark/actions/ExpireSnapshotsSparkAction.java | 270 --
.../org/apache/iceberg/spark/actions/FileInfo.java | 52 -
.../iceberg/spark/actions/ManifestFileBean.java | 180 --
.../spark/actions/MigrateTableSparkAction.java | 259 --
.../iceberg/spark/actions/NDVSketchUtil.java | 91 -
.../actions/RemoveDanglingDeletesSparkAction.java | 205 --
.../spark/actions/RewriteDataFilesSparkAction.java | 435 ---
.../spark/actions/RewriteManifestsSparkAction.java | 553 ----
.../RewritePositionDeleteFilesSparkAction.java | 377 ---
.../spark/actions/RewriteTablePathSparkAction.java | 748 -----
.../iceberg/spark/actions/SetAccumulator.java | 61 -
.../spark/actions/SnapshotTableSparkAction.java | 225 --
.../apache/iceberg/spark/actions/SparkActions.java | 116 -
.../actions/SparkBinPackFileRewriteRunner.java | 71 -
.../spark/actions/SparkDataFileRewriteRunner.java | 61 -
.../actions/SparkRewritePositionDeleteRunner.java | 150 -
.../iceberg/spark/actions/SparkRewriteRunner.java | 82 -
.../actions/SparkShufflingDataRewritePlanner.java | 82 -
.../actions/SparkShufflingFileRewriteRunner.java | 228 --
.../spark/actions/SparkSortFileRewriteRunner.java | 64 -
.../actions/SparkZOrderFileRewriteRunner.java | 204 --
.../iceberg/spark/actions/SparkZOrderUDF.java | 325 ---
.../spark/data/AvroWithSparkSchemaVisitor.java | 76 -
.../spark/data/ParquetWithSparkSchemaVisitor.java | 223 --
.../apache/iceberg/spark/data/SparkAvroWriter.java | 165 --
.../apache/iceberg/spark/data/SparkOrcReader.java | 135 -
.../iceberg/spark/data/SparkOrcValueReaders.java | 261 --
.../iceberg/spark/data/SparkOrcValueWriters.java | 221 --
.../apache/iceberg/spark/data/SparkOrcWriter.java | 236 --
.../iceberg/spark/data/SparkParquetReaders.java | 744 -----
.../iceberg/spark/data/SparkParquetWriters.java | 571 ----
.../iceberg/spark/data/SparkPlannedAvroReader.java | 192 --
.../iceberg/spark/data/SparkValueReaders.java | 322 ---
.../iceberg/spark/data/SparkValueWriters.java | 256 --
.../vectorized/ArrowVectorAccessorFactory.java | 132 -
.../data/vectorized/ArrowVectorAccessors.java | 38 -
.../spark/data/vectorized/ColumnVectorBuilder.java | 45 -
.../data/vectorized/ColumnVectorWithFilter.java | 155 -
.../spark/data/vectorized/ColumnarBatchReader.java | 147 -
.../spark/data/vectorized/ColumnarBatchUtil.java | 181 --
.../spark/data/vectorized/CometColumnReader.java | 140 -
.../data/vectorized/CometColumnarBatchReader.java | 197 --
.../data/vectorized/CometConstantColumnReader.java | 65 -
.../data/vectorized/CometDeleteColumnReader.java | 75 -
.../data/vectorized/CometPositionColumnReader.java | 62 -
.../vectorized/CometVectorizedReaderBuilder.java | 147 -
.../data/vectorized/ConstantColumnVector.java | 141 -
.../spark/data/vectorized/DeletedColumnVector.java | 122 -
.../data/vectorized/IcebergArrowColumnVector.java | 163 --
.../data/vectorized/RowPositionColumnVector.java | 120 -
.../data/vectorized/VectorizedSparkOrcReaders.java | 497 ----
.../vectorized/VectorizedSparkParquetReaders.java | 152 -
.../spark/functions/BaseScalarFunction.java | 40 -
.../iceberg/spark/functions/BucketFunction.java | 330 ---
.../iceberg/spark/functions/DaysFunction.java | 144 -
.../iceberg/spark/functions/HoursFunction.java | 125 -
.../spark/functions/IcebergVersionFunction.java | 95 -
.../iceberg/spark/functions/MonthsFunction.java | 143 -
.../iceberg/spark/functions/SparkFunctions.java | 75 -
.../iceberg/spark/functions/TruncateFunction.java | 356 ---
.../spark/functions/UnaryUnboundFunction.java | 44 -
.../iceberg/spark/functions/YearsFunction.java | 143 -
.../spark/procedures/AddFilesProcedure.java | 266 --
.../spark/procedures/AncestorsOfProcedure.java | 115 -
.../iceberg/spark/procedures/BaseProcedure.java | 240 --
.../procedures/CherrypickSnapshotProcedure.java | 100 -
.../procedures/ComputeTableStatsProcedure.java | 122 -
.../procedures/CreateChangelogViewProcedure.java | 321 ---
.../spark/procedures/ExpireSnapshotsProcedure.java | 173 --
.../procedures/FastForwardBranchProcedure.java | 94 -
.../spark/procedures/MigrateTableProcedure.java | 123 -
.../iceberg/spark/procedures/ProcedureInput.java | 241 --
.../spark/procedures/PublishChangesProcedure.java | 115 -
.../spark/procedures/RegisterTableProcedure.java | 110 -
.../procedures/RemoveOrphanFilesProcedure.java | 222 --
.../procedures/RewriteDataFilesProcedure.java | 215 --
.../procedures/RewriteManifestsProcedure.java | 121 -
.../RewritePositionDeleteFilesProcedure.java | 122 -
.../procedures/RewriteTablePathProcedure.java | 130 -
.../procedures/RollbackToSnapshotProcedure.java | 99 -
.../procedures/RollbackToTimestampProcedure.java | 104 -
.../procedures/SetCurrentSnapshotProcedure.java | 116 -
.../spark/procedures/SnapshotTableProcedure.java | 122 -
.../iceberg/spark/procedures/SparkProcedures.java | 74 -
.../iceberg/spark/source/BaseBatchReader.java | 140 -
.../apache/iceberg/spark/source/BaseReader.java | 246 --
.../apache/iceberg/spark/source/BaseRowReader.java | 125 -
.../iceberg/spark/source/BatchDataReader.java | 118 -
.../iceberg/spark/source/ChangelogRowReader.java | 173 --
.../apache/iceberg/spark/source/DVIterator.java | 109 -
.../spark/source/EqualityDeleteRowReader.java | 56 -
.../iceberg/spark/source/HasIcebergCatalog.java | 30 -
.../apache/iceberg/spark/source/IcebergSource.java | 255 --
.../iceberg/spark/source/InternalRowWrapper.java | 109 -
.../spark/source/PositionDeletesRowReader.java | 117 -
.../apache/iceberg/spark/source/RowDataReader.java | 122 -
.../spark/source/SerializableTableWithSize.java | 108 -
.../apache/iceberg/spark/source/SparkBatch.java | 229 --
.../iceberg/spark/source/SparkBatchQueryScan.java | 293 --
.../iceberg/spark/source/SparkChangelogScan.java | 168 --
.../iceberg/spark/source/SparkChangelogTable.java | 120 -
.../iceberg/spark/source/SparkCleanupUtil.java | 138 -
.../spark/source/SparkColumnStatistics.java | 88 -
.../spark/source/SparkColumnarReaderFactory.java | 71 -
.../spark/source/SparkCopyOnWriteOperation.java | 107 -
.../iceberg/spark/source/SparkCopyOnWriteScan.java | 190 --
.../spark/source/SparkFileWriterFactory.java | 296 --
.../iceberg/spark/source/SparkInputPartition.java | 99 -
.../iceberg/spark/source/SparkLocalScan.java | 66 -
.../iceberg/spark/source/SparkMetadataColumn.java | 50 -
.../spark/source/SparkMicroBatchStream.java | 521 ----
.../spark/source/SparkPartitionedFanoutWriter.java | 55 -
.../spark/source/SparkPartitionedWriter.java | 55 -
.../spark/source/SparkPartitioningAwareScan.java | 253 --
.../iceberg/spark/source/SparkPlanningUtil.java | 93 -
.../spark/source/SparkPositionDeletesRewrite.java | 522 ----
.../source/SparkPositionDeletesRewriteBuilder.java | 108 -
.../spark/source/SparkPositionDeltaOperation.java | 115 -
.../spark/source/SparkPositionDeltaWrite.java | 873 ------
.../source/SparkPositionDeltaWriteBuilder.java | 116 -
.../source/SparkRowLevelOperationBuilder.java | 114 -
.../spark/source/SparkRowReaderFactory.java | 68 -
.../org/apache/iceberg/spark/source/SparkScan.java | 354 ---
.../iceberg/spark/source/SparkScanBuilder.java | 762 -----
.../iceberg/spark/source/SparkStagedScan.java | 98 -
.../spark/source/SparkStagedScanBuilder.java | 92 -
.../apache/iceberg/spark/source/SparkTable.java | 453 ---
.../org/apache/iceberg/spark/source/SparkView.java | 151 -
.../apache/iceberg/spark/source/SparkWrite.java | 807 ------
.../iceberg/spark/source/SparkWriteBuilder.java | 203 --
.../iceberg/spark/source/StagedSparkTable.java | 41 -
.../org/apache/iceberg/spark/source/Stats.java | 52 -
.../iceberg/spark/source/StreamingOffset.java | 157 -
.../iceberg/spark/source/StructInternalRow.java | 385 ---
.../spark/source/metrics/EqualityDeleteFiles.java | 36 -
.../spark/source/metrics/IndexedDeleteFiles.java | 36 -
.../iceberg/spark/source/metrics/NumDeletes.java | 47 -
.../iceberg/spark/source/metrics/NumSplits.java | 45 -
.../source/metrics/PositionalDeleteFiles.java | 36 -
.../spark/source/metrics/ResultDataFiles.java | 36 -
.../spark/source/metrics/ResultDeleteFiles.java | 36 -
.../spark/source/metrics/ScannedDataManifests.java | 36 -
.../source/metrics/ScannedDeleteManifests.java | 36 -
.../spark/source/metrics/SkippedDataFiles.java | 36 -
.../spark/source/metrics/SkippedDataManifests.java | 36 -
.../spark/source/metrics/SkippedDeleteFiles.java | 36 -
.../source/metrics/SkippedDeleteManifests.java | 36 -
.../source/metrics/TaskEqualityDeleteFiles.java | 47 -
.../source/metrics/TaskIndexedDeleteFiles.java | 47 -
.../spark/source/metrics/TaskNumDeletes.java | 39 -
.../spark/source/metrics/TaskNumSplits.java | 39 -
.../source/metrics/TaskPositionalDeleteFiles.java | 47 -
.../spark/source/metrics/TaskResultDataFiles.java | 47 -
.../source/metrics/TaskResultDeleteFiles.java | 47 -
.../source/metrics/TaskScannedDataManifests.java | 47 -
.../source/metrics/TaskScannedDeleteManifests.java | 47 -
.../spark/source/metrics/TaskSkippedDataFiles.java | 47 -
.../source/metrics/TaskSkippedDataManifests.java | 47 -
.../source/metrics/TaskSkippedDeleteFiles.java | 47 -
.../source/metrics/TaskSkippedDeleteManifests.java | 47 -
.../source/metrics/TaskTotalDataFileSize.java | 48 -
.../source/metrics/TaskTotalDataManifests.java | 47 -
.../source/metrics/TaskTotalDeleteFileSize.java | 48 -
.../source/metrics/TaskTotalDeleteManifests.java | 47 -
.../source/metrics/TaskTotalPlanningDuration.java | 48 -
.../spark/source/metrics/TotalDataFileSize.java | 36 -
.../spark/source/metrics/TotalDataManifests.java | 36 -
.../spark/source/metrics/TotalDeleteFileSize.java | 36 -
.../spark/source/metrics/TotalDeleteManifests.java | 36 -
.../source/metrics/TotalPlanningDuration.java | 36 -
.../analysis/IcebergAnalysisException.java | 37 -
.../analysis/NoSuchProcedureException.java | 38 -
.../sql/connector/iceberg/catalog/Procedure.java | 50 -
.../iceberg/catalog/ProcedureCatalog.java | 40 -
.../iceberg/catalog/ProcedureParameter.java | 56 -
.../iceberg/catalog/ProcedureParameterImpl.java | 75 -
...org.apache.spark.sql.sources.DataSourceRegister | 20 -
.../plans/logical/OrderAwareCoalesce.scala | 51 -
.../logical/SetWriteDistributionAndOrdering.scala | 44 -
.../plans/logical/SortOrderParserUtil.scala | 40 -
.../spark/sql/catalyst/utils/PlanUtils.scala | 42 -
.../sql/execution/OrderAwareCoalesceExec.scala | 59 -
.../datasources/SparkExpressionConverter.scala | 66 -
.../apache/spark/sql/stats/ThetaSketchAgg.scala | 122 -
.../test/java/org/apache/iceberg/KryoHelpers.java | 51 -
.../iceberg/SparkDistributedDataScanTestBase.java | 95 -
.../java/org/apache/iceberg/TaskCheckHelper.java | 108 -
.../apache/iceberg/TestDataFileSerialization.java | 175 --
.../apache/iceberg/TestFileIOSerialization.java | 106 -
.../TestHadoopMetricsContextSerialization.java | 56 -
.../iceberg/TestManifestFileSerialization.java | 221 --
.../apache/iceberg/TestScanTaskSerialization.java | 197 --
.../TestSparkDistributedDataScanDeletes.java | 91 -
.../TestSparkDistributedDataScanFilterFiles.java | 85 -
...tSparkDistributedDataScanJavaSerialization.java | 40 -
...tSparkDistributedDataScanKryoSerialization.java | 40 -
.../TestSparkDistributedDataScanReporting.java | 86 -
.../org/apache/iceberg/TestTableSerialization.java | 173 --
.../java/org/apache/iceberg/ValidationHelpers.java | 78 -
.../org/apache/iceberg/spark/CatalogTestBase.java | 59 -
.../java/org/apache/iceberg/spark/Employee.java | 66 -
.../apache/iceberg/spark/SparkCatalogConfig.java | 87 -
.../apache/iceberg/spark/SparkTestHelperBase.java | 89 -
.../spark/SystemFunctionPushDownHelper.java | 127 -
.../java/org/apache/iceberg/spark/TestBase.java | 287 --
.../apache/iceberg/spark/TestBaseWithCatalog.java | 195 --
.../iceberg/spark/TestChangelogIterator.java | 359 ---
.../iceberg/spark/TestFileRewriteCoordinator.java | 275 --
.../apache/iceberg/spark/TestFunctionCatalog.java | 136 -
.../org/apache/iceberg/spark/TestSpark3Util.java | 186 --
.../iceberg/spark/TestSparkCachedTableCatalog.java | 102 -
.../iceberg/spark/TestSparkCatalogOperations.java | 192 --
.../iceberg/spark/TestSparkCompressionUtil.java | 162 --
.../TestSparkDistributionAndOrderingUtil.java | 3026 --------------------
.../iceberg/spark/TestSparkExecutorCache.java | 510 ----
.../org/apache/iceberg/spark/TestSparkFilters.java | 189 --
.../apache/iceberg/spark/TestSparkSchemaUtil.java | 84 -
.../iceberg/spark/TestSparkSessionCatalog.java | 101 -
.../apache/iceberg/spark/TestSparkTableUtil.java | 100 -
.../apache/iceberg/spark/TestSparkV2Filters.java | 834 ------
.../iceberg/spark/TestSparkValueConverter.java | 94 -
.../apache/iceberg/spark/TestSparkWriteConf.java | 592 ----
.../spark/actions/TestComputeTableStatsAction.java | 443 ---
.../iceberg/spark/actions/TestCreateActions.java | 1056 -------
.../actions/TestDeleteReachableFilesAction.java | 404 ---
.../spark/actions/TestExpireSnapshotsAction.java | 1345 ---------
.../spark/actions/TestMigrateTableAction.java | 68 -
.../actions/TestRemoveDanglingDeleteAction.java | 540 ----
.../spark/actions/TestRemoveOrphanFilesAction.java | 1093 -------
.../actions/TestRemoveOrphanFilesAction3.java | 194 --
.../spark/actions/TestRewriteDataFilesAction.java | 2403 ----------------
.../spark/actions/TestRewriteManifestsAction.java | 1201 --------
.../TestRewritePositionDeleteFilesAction.java | 1287 ---------
.../spark/actions/TestRewriteTablePathsAction.java | 1266 --------
.../spark/actions/TestSnapshotTableAction.java | 68 -
.../spark/actions/TestSparkFileRewriteRunners.java | 139 -
.../TestSparkShufflingDataRewritePlanner.java | 92 -
.../apache/iceberg/spark/data/AvroDataTest.java | 608 ----
.../apache/iceberg/spark/data/GenericsHelpers.java | 406 ---
.../org/apache/iceberg/spark/data/RandomData.java | 381 ---
.../org/apache/iceberg/spark/data/TestHelpers.java | 910 ------
.../apache/iceberg/spark/data/TestOrcWrite.java | 59 -
.../iceberg/spark/data/TestParquetAvroReader.java | 237 --
.../iceberg/spark/data/TestParquetAvroWriter.java | 123 -
.../iceberg/spark/data/TestSparkAvroEnums.java | 97 -
.../iceberg/spark/data/TestSparkAvroReader.java | 74 -
.../iceberg/spark/data/TestSparkDateTimes.java | 78 -
.../data/TestSparkOrcReadMetadataColumns.java | 219 --
.../iceberg/spark/data/TestSparkOrcReader.java | 110 -
.../data/TestSparkParquetReadMetadataColumns.java | 308 --
.../iceberg/spark/data/TestSparkParquetReader.java | 251 --
.../iceberg/spark/data/TestSparkParquetWriter.java | 154 -
.../spark/data/TestSparkRecordOrcReaderWriter.java | 153 -
.../spark/data/TestVectorizedOrcDataReader.java | 166 --
...estParquetDictionaryEncodedVectorizedReads.java | 188 --
...naryFallbackToPlainEncodingVectorizedReads.java | 77 -
.../vectorized/TestParquetVectorizedReads.java | 332 ---
.../spark/functions/TestSparkFunctions.java | 158 -
.../apache/iceberg/spark/source/ComplexRecord.java | 74 -
.../spark/source/DataFrameWriteTestBase.java | 147 -
.../spark/source/FilePathLastModifiedRecord.java | 80 -
.../iceberg/spark/source/FourColumnRecord.java | 106 -
.../apache/iceberg/spark/source/LogMessage.java | 119 -
.../apache/iceberg/spark/source/ManualSource.java | 75 -
.../apache/iceberg/spark/source/NestedRecord.java | 77 -
.../apache/iceberg/spark/source/ScanTestBase.java | 138 -
.../apache/iceberg/spark/source/SimpleRecord.java | 78 -
.../spark/source/SparkSQLExecutionHelper.java | 70 -
.../spark/source/TestAvroDataFrameWrite.java | 33 -
.../apache/iceberg/spark/source/TestAvroScan.java | 59 -
.../iceberg/spark/source/TestBaseReader.java | 289 --
.../iceberg/spark/source/TestChangelogReader.java | 263 --
.../spark/source/TestCompressionSettings.java | 276 --
.../spark/source/TestDataFrameWriterV2.java | 387 ---
.../source/TestDataFrameWriterV2Coercion.java | 81 -
.../spark/source/TestDataSourceOptions.java | 521 ----
.../iceberg/spark/source/TestFilteredScan.java | 666 -----
.../spark/source/TestForwardCompatibility.java | 224 --
.../iceberg/spark/source/TestIcebergSource.java | 42 -
.../source/TestIcebergSourceHadoopTables.java | 68 -
.../spark/source/TestIcebergSourceHiveTables.java | 86 -
.../spark/source/TestIcebergSourceTablesBase.java | 2468 ----------------
.../iceberg/spark/source/TestIcebergSpark.java | 241 --
.../spark/source/TestIdentityPartitionData.java | 250 --
.../spark/source/TestInternalRowWrapper.java | 81 -
.../source/TestMetadataTableReadableMetrics.java | 380 ---
.../TestMetadataTablesWithPartitionEvolution.java | 725 -----
.../spark/source/TestORCDataFrameWrite.java | 33 -
.../spark/source/TestParquetDataFrameWrite.java | 33 -
.../iceberg/spark/source/TestParquetScan.java | 86 -
.../spark/source/TestParquetVectorizedScan.java | 26 -
.../iceberg/spark/source/TestPartitionPruning.java | 485 ----
.../iceberg/spark/source/TestPartitionValues.java | 509 ----
.../iceberg/spark/source/TestPathIdentifier.java | 83 -
.../spark/source/TestPositionDeletesReader.java | 329 ---
.../spark/source/TestPositionDeletesTable.java | 1779 ------------
.../iceberg/spark/source/TestReadProjection.java | 640 -----
.../TestRequiredDistributionAndOrdering.java | 307 --
.../iceberg/spark/source/TestRuntimeFiltering.java | 510 ----
.../spark/source/TestSnapshotSelection.java | 586 ----
.../iceberg/spark/source/TestSparkAggregates.java | 84 -
.../iceberg/spark/source/TestSparkCatalog.java | 68 -
.../source/TestSparkCatalogCacheExpiration.java | 152 -
.../source/TestSparkCatalogHadoopOverrides.java | 140 -
.../iceberg/spark/source/TestSparkDVWriters.java | 69 -
.../iceberg/spark/source/TestSparkDataFile.java | 345 ---
.../iceberg/spark/source/TestSparkDataWrite.java | 742 -----
.../spark/source/TestSparkFileWriterFactory.java | 69 -
.../spark/source/TestSparkMetadataColumns.java | 367 ---
.../spark/source/TestSparkPartitioningWriters.java | 69 -
.../spark/source/TestSparkPlanningUtil.java | 213 --
.../source/TestSparkPositionDeltaWriters.java | 69 -
.../iceberg/spark/source/TestSparkReadMetrics.java | 180 --
.../spark/source/TestSparkReadProjection.java | 268 --
.../spark/source/TestSparkReaderDeletes.java | 752 -----
.../source/TestSparkReaderWithBloomFilter.java | 377 ---
.../spark/source/TestSparkRollingFileWriters.java | 55 -
.../apache/iceberg/spark/source/TestSparkScan.java | 1089 -------
.../iceberg/spark/source/TestSparkStagedScan.java | 125 -
.../iceberg/spark/source/TestSparkTable.java | 56 -
.../spark/source/TestSparkWriterMetrics.java | 67 -
.../iceberg/spark/source/TestStreamingOffset.java | 56 -
.../spark/source/TestStructuredStreaming.java | 310 --
.../spark/source/TestStructuredStreamingRead3.java | 779 -----
.../apache/iceberg/spark/source/TestTables.java | 209 --
.../spark/source/TestTimestampWithoutZone.java | 232 --
.../spark/source/TestWriteMetricsConfig.java | 305 --
.../iceberg/spark/source/ThreeColumnRecord.java | 83 -
.../spark/sql/PartitionedWritesTestBase.java | 282 --
.../iceberg/spark/sql/TestAggregatePushDown.java | 866 ------
.../apache/iceberg/spark/sql/TestAlterTable.java | 324 ---
.../apache/iceberg/spark/sql/TestCreateTable.java | 457 ---
.../iceberg/spark/sql/TestCreateTableAsSelect.java | 451 ---
.../apache/iceberg/spark/sql/TestDeleteFrom.java | 165 --
.../apache/iceberg/spark/sql/TestDropTable.java | 158 -
.../iceberg/spark/sql/TestFilterPushDown.java | 616 ----
.../apache/iceberg/spark/sql/TestNamespaceSQL.java | 302 --
.../iceberg/spark/sql/TestPartitionedWrites.java | 21 -
.../spark/sql/TestPartitionedWritesAsSelect.java | 158 -
.../spark/sql/TestPartitionedWritesToBranch.java | 49 -
.../sql/TestPartitionedWritesToWapBranch.java | 91 -
.../apache/iceberg/spark/sql/TestRefreshTable.java | 86 -
.../org/apache/iceberg/spark/sql/TestSelect.java | 616 ----
.../iceberg/spark/sql/TestSparkBucketFunction.java | 341 ---
.../iceberg/spark/sql/TestSparkDaysFunction.java | 107 -
.../iceberg/spark/sql/TestSparkHoursFunction.java | 93 -
.../iceberg/spark/sql/TestSparkMonthsFunction.java | 125 -
.../spark/sql/TestSparkTruncateFunction.java | 443 ---
.../iceberg/spark/sql/TestSparkYearsFunction.java | 125 -
.../spark/sql/TestStoragePartitionedJoins.java | 682 -----
.../spark/sql/TestTimestampWithoutZone.java | 243 --
.../iceberg/spark/sql/TestUnpartitionedWrites.java | 21 -
.../spark/sql/TestUnpartitionedWritesToBranch.java | 61 -
.../spark/sql/UnpartitionedWritesTestBase.java | 191 --
.../decimal_dict_and_plain_encoding.parquet | Bin 3685 -> 0 bytes
580 files changed, 10 insertions(+), 132207 deletions(-)
diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml
index 2c76595432..32b387c766 100644
--- a/.github/workflows/java-ci.yml
+++ b/.github/workflows/java-ci.yml
@@ -95,7 +95,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
- jvm: [17, 21]
+ jvm: [11, 17, 21]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
@@ -108,7 +108,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
- jvm: [17, 21]
+ jvm: [11, 17, 21]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
diff --git a/.github/workflows/jmh-benchmarks.yml b/.github/workflows/jmh-benchmarks.yml
index 6c0b100780..cfb53513e7 100644
--- a/.github/workflows/jmh-benchmarks.yml
+++ b/.github/workflows/jmh-benchmarks.yml
@@ -28,8 +28,8 @@ on:
description: 'The branch name'
required: true
spark_version:
- description: 'The spark project version to use, such as iceberg-spark-4.0'
- default: 'iceberg-spark-4.0'
+ description: 'The spark project version to use, such as iceberg-spark-3.5'
+ default: 'iceberg-spark-3.5'
required: true
benchmarks:
description: 'A list of comma-separated double-quoted Benchmark names, such as "IcebergSourceFlatParquetDataReadBenchmark", "IcebergSourceFlatParquetDataFilterBenchmark"'
diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
index d8b209e866..d7c680e74d 100644
--- a/.github/workflows/publish-snapshot.yml
+++ b/.github/workflows/publish-snapshot.yml
@@ -41,4 +41,4 @@ jobs:
- run: |
./gradlew printVersion
./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
- ./gradlew -DflinkVersions= -DsparkVersions=3.4,3.5,4.0 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
+ ./gradlew -DflinkVersions= -DsparkVersions=3.4,3.5 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
diff --git a/.github/workflows/recurring-jmh-benchmarks.yml b/.github/workflows/recurring-jmh-benchmarks.yml
index eec1a4aac1..71a52640b2 100644
--- a/.github/workflows/recurring-jmh-benchmarks.yml
+++ b/.github/workflows/recurring-jmh-benchmarks.yml
@@ -41,7 +41,7 @@ jobs:
"IcebergSourceNestedParquetDataReadBenchmark", "IcebergSourceNestedParquetDataWriteBenchmark",
"IcebergSourceParquetEqDeleteBenchmark", "IcebergSourceParquetMultiDeleteFileBenchmark",
"IcebergSourceParquetPosDeleteBenchmark", "IcebergSourceParquetWithUnrelatedDeleteBenchmark"]
- spark_version: ['iceberg-spark-4.0']
+ spark_version: ['iceberg-spark-3.5']
env:
SPARK_LOCAL_IP: localhost
steps:
diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml
index 10f468cb5a..a34bba0c82 100644
--- a/.github/workflows/spark-ci.yml
+++ b/.github/workflows/spark-ci.yml
@@ -71,17 +71,13 @@ jobs:
strategy:
matrix:
jvm: [11, 17, 21]
- spark: ['3.4', '3.5', '4.0']
+ spark: ['3.4', '3.5']
scala: ['2.12', '2.13']
exclude:
# Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
# Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831)
- - jvm: 11
- spark: '4.0'
- jvm: 21
spark: '3.4'
- - spark: '4.0'
- scala: '2.12'
env:
SPARK_LOCAL_IP: localhost
steps:
diff --git a/.gitignore b/.gitignore
index 2bebdd4a91..cbe36d543f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,8 +35,6 @@ spark/v3.4/spark/benchmark/*
spark/v3.4/spark-extensions/benchmark/*
spark/v3.5/spark/benchmark/*
spark/v3.5/spark-extensions/benchmark/*
-spark/v4.0/spark/benchmark/*
-spark/v4.0/spark-extensions/benchmark/*
*/benchmark/*
__pycache__/
diff --git a/build.gradle b/build.gradle
index 50a1029a30..838270f7e8 100644
--- a/build.gradle
+++ b/build.gradle
@@ -120,9 +120,6 @@ allprojects {
repositories {
mavenCentral()
mavenLocal()
- maven {
- url "https://repository.apache.org/content/repositories/orgapachespark-1480/"
- }
}
}
diff --git a/core/src/main/java/org/apache/iceberg/MetadataColumns.java b/core/src/main/java/org/apache/iceberg/MetadataColumns.java
index 5716eb9f1b..7b26290076 100644
--- a/core/src/main/java/org/apache/iceberg/MetadataColumns.java
+++ b/core/src/main/java/org/apache/iceberg/MetadataColumns.java
@@ -51,7 +51,7 @@ public class MetadataColumns {
public static final int SPEC_ID_COLUMN_ID = Integer.MAX_VALUE - 4;
public static final String SPEC_ID_COLUMN_DOC = "Spec ID used to track the file containing a row";
public static final NestedField SPEC_ID =
- NestedField.optional(
+ NestedField.required(
SPEC_ID_COLUMN_ID, "_spec_id", Types.IntegerType.get(), SPEC_ID_COLUMN_DOC);
// the partition column type is not static and depends on all specs in the table
public static final int PARTITION_COLUMN_ID = Integer.MAX_VALUE - 5;
diff --git a/gradle.properties b/gradle.properties
index 4f70359bdd..b391971a95 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
jmhIncludeRegex=.*
systemProp.defaultFlinkVersions=2.0
systemProp.knownFlinkVersions=1.19,1.20,2.0
-systemProp.defaultSparkVersions=4.0
-systemProp.knownSparkVersions=3.4,3.5,4.0
+systemProp.defaultSparkVersions=3.5
+systemProp.knownSparkVersions=3.4,3.5
systemProp.defaultKafkaVersions=3
systemProp.knownKafkaVersions=3
systemProp.defaultScalaVersion=2.12
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index de233740d6..bfa9c74013 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -24,7 +24,6 @@ activation = "1.1.1"
aliyun-sdk-oss = "3.10.2"
analyticsaccelerator = "1.0.0"
antlr = "4.9.3"
-antlr413 = "4.13.1" # For Spark 4.0 support
aircompressor = "0.27"
apiguardian = "1.1.2"
arrow = "15.0.2"
@@ -83,7 +82,6 @@ slf4j = "2.0.17"
snowflake-jdbc = "3.24.0"
spark34 = "3.4.4"
spark35 = "3.5.5"
-spark40 = "4.0.0"
sqlite-jdbc = "3.49.1.0"
testcontainers = "1.21.0"
tez08 = { strictly = "0.8.4"} # see rich version usage explanation above
@@ -95,8 +93,6 @@ aliyun-sdk-oss = { module = "com.aliyun.oss:aliyun-sdk-oss", version.ref = "aliy
analyticsaccelerator-s3 = { module = "software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3", version.ref = "analyticsaccelerator" }
antlr-antlr4 = { module = "org.antlr:antlr4", version.ref = "antlr" }
antlr-runtime = { module = "org.antlr:antlr4-runtime", version.ref = "antlr" }
-antlr-antlr413 = { module = "org.antlr:antlr4", version.ref = "antlr413" }
-antlr-runtime413 = { module = "org.antlr:antlr4-runtime", version.ref = "antlr413" }
arrow-memory-netty = { module = "org.apache.arrow:arrow-memory-netty", version.ref = "arrow" }
arrow-vector = { module = "org.apache.arrow:arrow-vector", version.ref = "arrow" }
avro-avro = { module = "org.apache.avro:avro", version.ref = "avro" }
diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
index 7c1b91176f..9736b32e87 100644
--- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
+++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
@@ -282,7 +282,6 @@ public class TestHiveMetastore {
// Setting this to avoid thrift exception during running Iceberg tests outside Iceberg.
conf.set(
HiveConf.ConfVars.HIVE_IN_TEST.varname, HiveConf.ConfVars.HIVE_IN_TEST.getDefaultValue());
- conf.set("datanucleus.connectionPoolingType", "DBCP");
}
private static void setupMetastoreDB(String dbURL) throws SQLException, IOException {
diff --git a/jmh.gradle b/jmh.gradle
index 4d6d7207c5..c4efdc0da3 100644
--- a/jmh.gradle
+++ b/jmh.gradle
@@ -48,11 +48,6 @@ if (sparkVersions.contains("3.5")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-3.5_${scalaVersion}"))
}
-if (sparkVersions.contains("4.0")) {
- jmhProjects.add(project(":iceberg-spark:iceberg-spark-4.0_2.13"))
- jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.0_2.13"))
-}
-
configure(jmhProjects) {
apply plugin: 'me.champeau.jmh'
apply plugin: 'io.morethan.jmhreport'
diff --git a/settings.gradle b/settings.gradle
index e2bbe08a98..c47c887228 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -161,18 +161,6 @@ if (sparkVersions.contains("3.5")) {
project(":iceberg-spark:spark-runtime-3.5_${scalaVersion}").name = "iceberg-spark-runtime-3.5_${scalaVersion}"
}
-if (sparkVersions.contains("4.0")) {
- include ":iceberg-spark:spark-4.0_2.13"
- include ":iceberg-spark:spark-extensions-4.0_2.13"
- include ":iceberg-spark:spark-runtime-4.0_2.13"
- project(":iceberg-spark:spark-4.0_2.13").projectDir = file('spark/v4.0/spark')
- project(":iceberg-spark:spark-4.0_2.13").name = "iceberg-spark-4.0_2.13"
- project(":iceberg-spark:spark-extensions-4.0_2.13").projectDir = file('spark/v4.0/spark-extensions')
- project(":iceberg-spark:spark-extensions-4.0_2.13").name = "iceberg-spark-extensions-4.0_2.13"
- project(":iceberg-spark:spark-runtime-4.0_2.13").projectDir = file('spark/v4.0/spark-runtime')
- project(":iceberg-spark:spark-runtime-4.0_2.13").name = "iceberg-spark-runtime-4.0_2.13"
-}
-
if (kafkaVersions.contains("3")) {
include 'kafka-connect'
project(':kafka-connect').name = 'iceberg-kafka-connect'
diff --git a/spark/build.gradle b/spark/build.gradle
index 75d3f899e5..88d2524280 100644
--- a/spark/build.gradle
+++ b/spark/build.gradle
@@ -27,7 +27,3 @@ if (sparkVersions.contains("3.4")) {
if (sparkVersions.contains("3.5")) {
apply from: file("$projectDir/v3.5/build.gradle")
}
-
-if (sparkVersions.contains("4.0")) {
- apply from: file("$projectDir/v4.0/build.gradle")
-}
diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle
deleted file mode 100644
index 0be25bb268..0000000000
--- a/spark/v4.0/build.gradle
+++ /dev/null
@@ -1,337 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-String sparkMajorVersion = '4.0'
-String scalaVersion = '2.13'
-
-def sparkProjects = [
- project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}"),
- project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}"),
- project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}"),
-]
-
-configure(sparkProjects) {
- configurations {
- all {
- resolutionStrategy {
- force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson215.get()}"
- force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson215.get()}"
- force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson215.get()}"
- }
- }
- }
-}
-
-project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
- apply plugin: 'scala'
- apply plugin: 'com.github.alisiikh.scalastyle'
-
- sourceSets {
- main {
- scala.srcDirs = ['src/main/scala', 'src/main/java']
- java.srcDirs = []
- }
- }
-
- dependencies {
- implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
- api project(':iceberg-api')
- annotationProcessor libs.immutables.value
- compileOnly libs.immutables.value
- implementation project(':iceberg-common')
- implementation project(':iceberg-core')
- implementation project(':iceberg-data')
- implementation project(':iceberg-orc')
- implementation project(':iceberg-parquet')
- implementation project(':iceberg-arrow')
- implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
- implementation("org.apache.datasketches:datasketches-java:${libs.versions.datasketches.get()}")
-
- compileOnly libs.errorprone.annotations
- compileOnly libs.avro.avro
- compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark40.get()}") {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.apache.arrow'
- exclude group: 'org.apache.parquet'
- // to make sure netty libs only come from project(':iceberg-arrow')
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- exclude group: 'org.roaringbitmap'
- }
-
- compileOnly "org.apache.datafusion:comet-spark-spark3.5_2.13:${libs.versions.comet.get()}"
-
- implementation libs.parquet.column
- implementation libs.parquet.hadoop
-
- implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
- exclude group: 'org.apache.hadoop'
- exclude group: 'commons-lang'
- // These artifacts are shaded and included in the orc-core fat jar
- exclude group: 'com.google.protobuf', module: 'protobuf-java'
- exclude group: 'org.apache.hive', module: 'hive-storage-api'
- }
-
- implementation(libs.arrow.vector) {
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
-
- implementation libs.caffeine
-
- testImplementation(libs.hadoop3.minicluster) {
- exclude group: 'org.apache.avro', module: 'avro'
- // to make sure netty libs only come from project(':iceberg-arrow')
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- }
- testImplementation project(path: ':iceberg-hive-metastore')
- testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
- testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
- transitive = false
- }
- testImplementation libs.sqlite.jdbc
- testImplementation libs.awaitility
- // runtime dependencies for running REST Catalog based integration test
- testRuntimeOnly libs.jetty.servlet
- }
-
- test {
- useJUnitPlatform()
- }
-
- tasks.withType(Test) {
- // Vectorized reads need more memory
- maxHeapSize '2560m'
- }
-}
-
-project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") {
- apply plugin: 'java-library'
- apply plugin: 'scala'
- apply plugin: 'com.github.alisiikh.scalastyle'
- apply plugin: 'antlr'
-
- configurations {
- /*
- The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This
- bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate
- runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with
- the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add
- the runtime dependency back to only the runtime configuration manually.
- */
- implementation {
- extendsFrom = extendsFrom.findAll { it != configurations.antlr }
- }
- }
-
- dependencies {
- implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
- implementation libs.roaringbitmap
-
- compileOnly "org.scala-lang:scala-library"
- compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
- compileOnly project(':iceberg-api')
- compileOnly project(':iceberg-core')
- compileOnly project(':iceberg-common')
- compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
- compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark40.get()}") {
- exclude group: 'org.apache.avro', module: 'avro'
- exclude group: 'org.apache.arrow'
- exclude group: 'org.apache.parquet'
- // to make sure netty libs only come from project(':iceberg-arrow')
- exclude group: 'io.netty', module: 'netty-buffer'
- exclude group: 'io.netty', module: 'netty-common'
- exclude group: 'org.roaringbitmap'
- }
- compileOnly libs.errorprone.annotations
-
- testImplementation project(path: ':iceberg-data')
- testImplementation project(path: ':iceberg-parquet')
- testImplementation project(path: ':iceberg-hive-metastore')
- testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
- testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
- testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
- transitive = false
- }
- // runtime dependencies for running REST Catalog based integration test
- testRuntimeOnly libs.jetty.servlet
- testRuntimeOnly libs.sqlite.jdbc
- testImplementation libs.avro.avro
- testImplementation libs.parquet.hadoop
- testImplementation libs.awaitility
- testImplementation "org.apache.datafusion:comet-spark-spark3.5_2.13:${libs.versions.comet.get()}"
-
- // Required because we remove antlr plugin dependencies from the compile configuration, see note above
- runtimeOnly libs.antlr.runtime413
- antlr libs.antlr.antlr413
- }
-
- test {
- useJUnitPlatform()
- }
-
- generateGrammarSource {
- maxHeapSize = "64m"
- arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions']
- }
-}
-
-project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}") {
- apply plugin: 'com.gradleup.shadow'
-
- tasks.jar.dependsOn tasks.shadowJar
-
- sourceSets {
- integration {
- java.srcDir "$projectDir/src/integration/java"
- resources.srcDir "$projectDir/src/integration/resources"
- }
- }
-
- configurations {
- implementation {
- exclude group: 'org.apache.spark'
- // included in Spark
- exclude group: 'org.slf4j'
- exclude group: 'org.apache.commons'
- exclude group: 'commons-pool'
- exclude group: 'commons-codec'
- exclude group: 'org.xerial.snappy'
- exclude group: 'javax.xml.bind'
- exclude group: 'javax.annotation'
- exclude group: 'com.github.luben'
- exclude group: 'com.ibm.icu'
- exclude group: 'org.glassfish'
- exclude group: 'org.abego.treelayout'
- exclude group: 'org.antlr'
- exclude group: 'org.scala-lang'
- exclude group: 'org.scala-lang.modules'
- }
- }
-
- dependencies {
- api project(':iceberg-api')
- implementation project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
- implementation project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
- implementation project(':iceberg-aws')
- implementation project(':iceberg-azure')
- implementation(project(':iceberg-aliyun')) {
- exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
- exclude group: 'org.apache.httpcomponents', module: 'httpclient'
- exclude group: 'commons-logging', module: 'commons-logging'
- }
- implementation project(':iceberg-gcp')
- implementation project(':iceberg-hive-metastore')
- implementation(project(':iceberg-nessie')) {
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
- implementation (project(':iceberg-snowflake')) {
- exclude group: 'net.snowflake' , module: 'snowflake-jdbc'
- }
-
- integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
- integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark40.get()}"
- integrationImplementation libs.junit.vintage.engine
- integrationImplementation libs.junit.jupiter
- integrationImplementation libs.junit.platform.launcher
- integrationImplementation libs.slf4j.simple
- integrationImplementation libs.assertj.core
- integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
- integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
- integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
- integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
-
- // runtime dependencies for running Hive Catalog based integration test
- integrationRuntimeOnly project(':iceberg-hive-metastore')
- // runtime dependencies for running REST Catalog based integration test
- integrationRuntimeOnly project(path: ':iceberg-core', configuration: 'testArtifacts')
- integrationRuntimeOnly (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
- transitive = false
- }
- integrationRuntimeOnly libs.jetty.servlet
- integrationRuntimeOnly libs.sqlite.jdbc
-
- // Not allowed on our classpath, only the runtime jar is allowed
- integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
- integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
- integrationCompileOnly project(':iceberg-api')
- }
-
- shadowJar {
- configurations = [project.configurations.runtimeClasspath]
-
- zip64 true
-
- // include the LICENSE and NOTICE files for the shaded Jar
- from(projectDir) {
- include 'LICENSE'
- include 'NOTICE'
- }
-
- // Relocate dependencies to avoid conflicts
- relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
- relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
- relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
- relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
- relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
- relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
- relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
- relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
- relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
- relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
- relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
- relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
- relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
- relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
- // relocate Arrow and related deps to shade Iceberg specific version
- relocate 'io.netty', 'org.apache.iceberg.shaded.io.netty'
- relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow'
- relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch'
- relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
- relocate 'org.roaringbitmap', 'org.apache.iceberg.shaded.org.roaringbitmap'
- relocate 'org.apache.datasketches', 'org.apache.iceberg.shaded.org.apache.datasketches'
-
- archiveClassifier.set(null)
- }
-
- task integrationTest(type: Test) {
- useJUnitPlatform()
- description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
- group = "verification"
- jvmArgs += project.property('extraJvmArgs')
- testClassesDirs = sourceSets.integration.output.classesDirs
- classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
- inputs.file(shadowJar.archiveFile.get().asFile.path)
- }
- integrationTest.dependsOn shadowJar
- check.dependsOn integrationTest
-
- jar {
- enabled = false
- }
-}
-
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/DeleteFileIndexBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/DeleteFileIndexBenchmark.java
deleted file mode 100644
index 9375ca3a4f..0000000000
--- a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/DeleteFileIndexBenchmark.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg;
-
-import com.google.errorprone.annotations.FormatMethod;
-import com.google.errorprone.annotations.FormatString;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.io.CloseableIterable;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.spark.Spark3Util;
-import org.apache.iceberg.spark.SparkSessionCatalog;
-import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
-import org.apache.iceberg.util.ThreadPools;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.catalyst.parser.ParseException;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Timeout;
-import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
-
-/**
- * A benchmark that evaluates the delete file index build and lookup performance.
- *
- * <p>To run this benchmark for spark-4.0: <code>
- * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.13:jmh
- * -PjmhIncludeRegex=DeleteFileIndexBenchmark
- * -PjmhOutputPath=benchmark/iceberg-delete-file-index-benchmark.txt
- * </code>
- */
-@Fork(1)
-@State(Scope.Benchmark)
-@Warmup(iterations = 3)
-@Measurement(iterations = 10)
-@Timeout(time = 20, timeUnit = TimeUnit.MINUTES)
-@BenchmarkMode(Mode.SingleShotTime)
-public class DeleteFileIndexBenchmark {
-
- private static final String TABLE_NAME = "test_table";
- private static final String PARTITION_COLUMN = "ss_ticket_number";
-
- private static final int NUM_PARTITIONS = 50;
- private static final int NUM_DATA_FILES_PER_PARTITION = 50_000;
- private static final int NUM_DELETE_FILES_PER_PARTITION = 100;
-
- private final Configuration hadoopConf = new Configuration();
- private SparkSession spark;
- private Table table;
-
- private List<DataFile> dataFiles;
-
- @Param({"partition", "file", "dv"})
- private String type;
-
- @Setup
- public void setupBenchmark() throws NoSuchTableException, ParseException {
- setupSpark();
- initTable();
- initDataAndDeletes();
- loadDataFiles();
- }
-
- private void initDataAndDeletes() {
- if (type.equals("partition")) {
- initDataAndPartitionScopedDeletes();
- } else if (type.equals("file")) {
- initDataAndFileScopedDeletes();
- } else {
- initDataAndDVs();
- }
- }
-
- @TearDown
- public void tearDownBenchmark() {
- dropTable();
- tearDownSpark();
- }
-
- @Benchmark
- @Threads(1)
- public void buildIndexAndLookup(Blackhole blackhole) {
- DeleteFileIndex deletes = buildDeletes();
- for (DataFile dataFile : dataFiles) {
- DeleteFile[] deleteFiles = deletes.forDataFile(dataFile.dataSequenceNumber(), dataFile);
- blackhole.consume(deleteFiles);
- }
- }
-
- private void loadDataFiles() {
- table.refresh();
-
- Snapshot snapshot = table.currentSnapshot();
-
- ManifestGroup manifestGroup =
- new ManifestGroup(table.io(), snapshot.dataManifests(table.io()), ImmutableList.of());
-
- try (CloseableIterable<ManifestEntry<DataFile>> entries = manifestGroup.entries()) {
- List<DataFile> files = Lists.newArrayList();
- for (ManifestEntry<DataFile> entry : entries) {
- files.add(entry.file().copyWithoutStats());
- }
- this.dataFiles = files;
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- private DeleteFileIndex buildDeletes() {
- table.refresh();
-
- List<ManifestFile> deleteManifests = table.currentSnapshot().deleteManifests(table.io());
-
- return DeleteFileIndex.builderFor(table.io(), deleteManifests)
- .specsById(table.specs())
- .planWith(ThreadPools.getWorkerPool())
- .build();
- }
-
- private void initDataAndPartitionScopedDeletes() {
- for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
- StructLike partition = TestHelpers.Row.of(partitionOrdinal);
-
- RowDelta rowDelta = table.newRowDelta();
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
- DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
- rowDelta.addRows(dataFile);
- }
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DELETE_FILES_PER_PARTITION; fileOrdinal++) {
- DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, partition);
- rowDelta.addDeletes(deleteFile);
- }
-
- rowDelta.commit();
- }
- }
-
- private void initDataAndFileScopedDeletes() {
- for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
- StructLike partition = TestHelpers.Row.of(partitionOrdinal);
-
- RowDelta rowDelta = table.newRowDelta();
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
- DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
- DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
- rowDelta.addRows(dataFile);
- rowDelta.addDeletes(deleteFile);
- }
-
- rowDelta.commit();
- }
- }
-
- private void initDataAndDVs() {
- for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
- StructLike partition = TestHelpers.Row.of(partitionOrdinal);
-
- RowDelta rowDelta = table.newRowDelta();
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
- DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
- DeleteFile dv = FileGenerationUtil.generateDV(table, dataFile);
- rowDelta.addRows(dataFile);
- rowDelta.addDeletes(dv);
- }
-
- rowDelta.commit();
- }
- }
-
- private void setupSpark() {
- this.spark =
- SparkSession.builder()
- .config("spark.ui.enabled", false)
- .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
- .config("spark.sql.catalog.spark_catalog.type", "hadoop")
- .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
- .master("local[*]")
- .getOrCreate();
- }
-
- private void tearDownSpark() {
- spark.stop();
- }
-
- private void initTable() throws NoSuchTableException, ParseException {
- sql(
- "CREATE TABLE %s ( "
- + " `ss_sold_date_sk` INT, "
- + " `ss_sold_time_sk` INT, "
- + " `ss_item_sk` INT, "
- + " `ss_customer_sk` STRING, "
- + " `ss_cdemo_sk` STRING, "
- + " `ss_hdemo_sk` STRING, "
- + " `ss_addr_sk` STRING, "
- + " `ss_store_sk` STRING, "
- + " `ss_promo_sk` STRING, "
- + " `ss_ticket_number` INT, "
- + " `ss_quantity` STRING, "
- + " `ss_wholesale_cost` STRING, "
- + " `ss_list_price` STRING, "
- + " `ss_sales_price` STRING, "
- + " `ss_ext_discount_amt` STRING, "
- + " `ss_ext_sales_price` STRING, "
- + " `ss_ext_wholesale_cost` STRING, "
- + " `ss_ext_list_price` STRING, "
- + " `ss_ext_tax` STRING, "
- + " `ss_coupon_amt` STRING, "
- + " `ss_net_paid` STRING, "
- + " `ss_net_paid_inc_tax` STRING, "
- + " `ss_net_profit` STRING "
- + ")"
- + "USING iceberg "
- + "PARTITIONED BY (%s) "
- + "TBLPROPERTIES ("
- + " '%s' '%b',"
- + " '%s' '%s',"
- + " '%s' '%d')",
- TABLE_NAME,
- PARTITION_COLUMN,
- TableProperties.MANIFEST_MERGE_ENABLED,
- false,
- TableProperties.DELETE_MODE,
- RowLevelOperationMode.MERGE_ON_READ.modeName(),
- TableProperties.FORMAT_VERSION,
- type.equals("dv") ? 3 : 2);
-
- this.table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
- }
-
- private void dropTable() {
- sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
- }
-
- private String newWarehouseDir() {
- return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
- }
-
- @FormatMethod
- private void sql(@FormatString String query, Object... args) {
- spark.sql(String.format(query, args));
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
deleted file mode 100644
index 963daa2c36..0000000000
--- a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark;
-
-import static org.apache.spark.sql.functions.current_date;
-import static org.apache.spark.sql.functions.date_add;
-import static org.apache.spark.sql.functions.expr;
-
-import com.google.errorprone.annotations.FormatMethod;
-import com.google.errorprone.annotations.FormatString;
-import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.DistributionMode;
-import org.apache.iceberg.RowLevelOperationMode;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.catalyst.parser.ParseException;
-import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.sql.types.StructType;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Warmup;
-
-/**
- * A benchmark that evaluates the performance of the cardinality check in MERGE operations.
- *
- * <p>To run this benchmark for spark-4.0: <code>
- * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.13:jmh
- * -PjmhIncludeRegex=MergeCardinalityCheckBenchmark
- * -PjmhOutputPath=benchmark/iceberg-merge-cardinality-check-benchmark.txt
- * </code>
- */
-@Fork(1)
-@State(Scope.Benchmark)
-@Warmup(iterations = 3)
-@Measurement(iterations = 5)
-@BenchmarkMode(Mode.SingleShotTime)
-public class MergeCardinalityCheckBenchmark {
-
- private static final String TABLE_NAME = "test_table";
- private static final int NUM_FILES = 5;
- private static final int NUM_ROWS_PER_FILE = 1_000_000;
- private static final int NUM_UNMATCHED_RECORDS_PER_MERGE = 100_000;
-
- private final Configuration hadoopConf = new Configuration();
- private SparkSession spark;
- private long originalSnapshotId;
-
- @Setup
- public void setupBenchmark() throws NoSuchTableException, ParseException {
- setupSpark();
- initTable();
- appendData();
-
- Table table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
- this.originalSnapshotId = table.currentSnapshot().snapshotId();
- }
-
- @TearDown
- public void tearDownBenchmark() {
- tearDownSpark();
- dropTable();
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteMergeCardinalityCheck10PercentUpdates() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.1);
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteMergeCardinalityCheck30PercentUpdates() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.3);
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteMergeCardinalityCheck90PercentUpdates() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.9);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadMergeCardinalityCheck10PercentUpdates() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.1);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadMergeCardinalityCheck30PercentUpdates() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.3);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadMergeCardinalityCheck90PercentUpdates() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.9);
- }
-
- private void runBenchmark(RowLevelOperationMode mode, double updatePercentage) {
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')",
- TABLE_NAME, TableProperties.MERGE_MODE, mode.modeName());
-
- Dataset<Long> insertDataDF = spark.range(-NUM_UNMATCHED_RECORDS_PER_MERGE, 0, 1);
- Dataset<Long> updateDataDF = spark.range((long) (updatePercentage * NUM_ROWS_PER_FILE));
- Dataset<Long> sourceDF = updateDataDF.union(insertDataDF);
- sourceDF.createOrReplaceTempView("source");
-
- sql(
- "MERGE INTO %s t USING source s "
- + "ON t.id = s.id "
- + "WHEN MATCHED THEN "
- + " UPDATE SET stringCol = 'invalid' "
- + "WHEN NOT MATCHED THEN "
- + " INSERT (id, intCol, floatCol, doubleCol, decimalCol, dateCol, timestampCol, stringCol) "
- + " VALUES (s.id, null, null, null, null, null, null, 'new')",
- TABLE_NAME);
-
- sql(
- "CALL system.rollback_to_snapshot(table => '%s', snapshot_id => %dL)",
- TABLE_NAME, originalSnapshotId);
- }
-
- private void setupSpark() {
- this.spark =
- SparkSession.builder()
- .config("spark.ui.enabled", false)
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
- .config("spark.sql.catalog.spark_catalog.type", "hadoop")
- .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
- .config(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key(), "false")
- .config(SQLConf.RUNTIME_ROW_LEVEL_OPERATION_GROUP_FILTER_ENABLED().key(), "false")
- .config(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false")
- .config(SQLConf.SHUFFLE_PARTITIONS().key(), "2")
- .master("local")
- .getOrCreate();
- }
-
- private void tearDownSpark() {
- spark.stop();
- }
-
- private void initTable() {
- sql(
- "CREATE TABLE %s ( "
- + " id LONG, intCol INT, floatCol FLOAT, doubleCol DOUBLE, "
- + " decimalCol DECIMAL(20, 5), dateCol DATE, timestampCol TIMESTAMP, "
- + " stringCol STRING)"
- + "USING iceberg "
- + "TBLPROPERTIES ("
- + " '%s' '%s',"
- + " '%s' '%d',"
- + " '%s' '%d')",
- TABLE_NAME,
- TableProperties.MERGE_DISTRIBUTION_MODE,
- DistributionMode.NONE.modeName(),
- TableProperties.SPLIT_OPEN_FILE_COST,
- Integer.MAX_VALUE,
- TableProperties.FORMAT_VERSION,
- 2);
-
- sql("ALTER TABLE %s WRITE ORDERED BY id", TABLE_NAME);
- }
-
- private void dropTable() {
- sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
- }
-
- private void appendData() throws NoSuchTableException {
- for (int fileNum = 1; fileNum <= NUM_FILES; fileNum++) {
- Dataset<Row> inputDF =
- spark
- .range(NUM_ROWS_PER_FILE)
- .withColumn("intCol", expr("CAST(id AS INT)"))
- .withColumn("floatCol", expr("CAST(id AS FLOAT)"))
- .withColumn("doubleCol", expr("CAST(id AS DOUBLE)"))
- .withColumn("decimalCol", expr("CAST(id AS DECIMAL(20, 5))"))
- .withColumn("dateCol", date_add(current_date(), fileNum))
- .withColumn("timestampCol", expr("TO_TIMESTAMP(dateCol)"))
- .withColumn("stringCol", expr("CAST(dateCol AS STRING)"));
- appendAsFile(inputDF);
- }
- }
-
- private void appendAsFile(Dataset<Row> df) throws NoSuchTableException {
- // ensure the schema is precise (including nullability)
- StructType sparkSchema = spark.table(TABLE_NAME).schema();
- spark.createDataFrame(df.rdd(), sparkSchema).coalesce(1).writeTo(TABLE_NAME).append();
- }
-
- private String newWarehouseDir() {
- return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
- }
-
- @FormatMethod
- private void sql(@FormatString String query, Object... args) {
- spark.sql(String.format(query, args));
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/PlanningBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/PlanningBenchmark.java
deleted file mode 100644
index 34d9d70e6c..0000000000
--- a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/PlanningBenchmark.java
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark;
-
-import static org.apache.iceberg.PlanningMode.DISTRIBUTED;
-import static org.apache.iceberg.PlanningMode.LOCAL;
-
-import com.google.errorprone.annotations.FormatMethod;
-import com.google.errorprone.annotations.FormatString;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.BatchScan;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.DeleteFile;
-import org.apache.iceberg.FileGenerationUtil;
-import org.apache.iceberg.PlanningMode;
-import org.apache.iceberg.RowDelta;
-import org.apache.iceberg.RowLevelOperationMode;
-import org.apache.iceberg.ScanTask;
-import org.apache.iceberg.SparkDistributedDataScan;
-import org.apache.iceberg.StructLike;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.TestHelpers;
-import org.apache.iceberg.expressions.Expression;
-import org.apache.iceberg.expressions.Expressions;
-import org.apache.iceberg.io.CloseableIterable;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
-import org.apache.iceberg.types.Conversions;
-import org.apache.iceberg.types.Types;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.catalyst.parser.ParseException;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Timeout;
-import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
-
-/**
- * A benchmark that evaluates the job planning performance.
- *
- * <p>To run this benchmark for spark-4.0: <code>
- * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.12:jmh
- * -PjmhIncludeRegex=PlanningBenchmark
- * -PjmhOutputPath=benchmark/iceberg-planning-benchmark.txt
- * </code>
- */
-@Fork(1)
-@State(Scope.Benchmark)
-@Warmup(iterations = 3)
-@Measurement(iterations = 5)
-@Timeout(time = 20, timeUnit = TimeUnit.MINUTES)
-@BenchmarkMode(Mode.SingleShotTime)
-public class PlanningBenchmark {
-
- private static final String TABLE_NAME = "test_table";
- private static final String PARTITION_COLUMN = "ss_ticket_number";
- private static final int PARTITION_VALUE = 10;
- private static final String SORT_KEY_COLUMN = "ss_sold_date_sk";
- private static final int SORT_KEY_VALUE = 5;
-
- private static final Expression SORT_KEY_PREDICATE =
- Expressions.equal(SORT_KEY_COLUMN, SORT_KEY_VALUE);
- private static final Expression PARTITION_PREDICATE =
- Expressions.equal(PARTITION_COLUMN, PARTITION_VALUE);
- private static final Expression PARTITION_AND_SORT_KEY_PREDICATE =
- Expressions.and(PARTITION_PREDICATE, SORT_KEY_PREDICATE);
-
- private static final int NUM_PARTITIONS = 30;
- private static final int NUM_DATA_FILES_PER_PARTITION = 50_000;
- private static final int NUM_DELETE_FILES_PER_PARTITION = 50;
-
- private final Configuration hadoopConf = new Configuration();
- private SparkSession spark;
- private Table table;
-
- @Param({"partition", "file", "dv"})
- private String type;
-
- @Setup
- public void setupBenchmark() throws NoSuchTableException, ParseException {
- setupSpark();
- initTable();
- initDataAndDeletes();
- }
-
- @TearDown
- public void tearDownBenchmark() {
- dropTable();
- tearDownSpark();
- }
-
- @Benchmark
- @Threads(1)
- public void localPlanningWithPartitionAndMinMaxFilter(Blackhole blackhole) {
- BatchScan scan = table.newBatchScan();
- List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, PARTITION_AND_SORT_KEY_PREDICATE);
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void distributedPlanningWithPartitionAndMinMaxFilter(Blackhole blackhole) {
- BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
- List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, PARTITION_AND_SORT_KEY_PREDICATE);
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void localPlanningWithMinMaxFilter(Blackhole blackhole) {
- BatchScan scan = table.newBatchScan();
- List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, SORT_KEY_PREDICATE);
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void distributedPlanningWithMinMaxFilter(Blackhole blackhole) {
- BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
- List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, SORT_KEY_PREDICATE);
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void localPlanningWithoutFilter(Blackhole blackhole) {
- BatchScan scan = table.newBatchScan();
- List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, Expressions.alwaysTrue());
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void distributedPlanningWithoutFilter(Blackhole blackhole) {
- BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
- List<ScanTask> fileTasks = planFilesWithoutColumnStats(scan, Expressions.alwaysTrue());
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void localPlanningWithoutFilterWithStats(Blackhole blackhole) {
- BatchScan scan = table.newBatchScan();
- List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void distributedPlanningWithoutFilterWithStats(Blackhole blackhole) {
- BatchScan scan = newDistributedScan(DISTRIBUTED, DISTRIBUTED);
- List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void distributedDataLocalDeletesPlanningWithoutFilterWithStats(Blackhole blackhole) {
- BatchScan scan = newDistributedScan(DISTRIBUTED, LOCAL);
- List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void localDataDistributedDeletesPlanningWithoutFilterWithStats(Blackhole blackhole) {
- BatchScan scan = newDistributedScan(LOCAL, DISTRIBUTED);
- List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
- blackhole.consume(fileTasks);
- }
-
- @Benchmark
- @Threads(1)
- public void localPlanningViaDistributedScanWithoutFilterWithStats(Blackhole blackhole) {
- BatchScan scan = newDistributedScan(LOCAL, LOCAL);
- List<ScanTask> fileTasks = planFilesWithColumnStats(scan, Expressions.alwaysTrue());
- blackhole.consume(fileTasks);
- }
-
- private void setupSpark() {
- this.spark =
- SparkSession.builder()
- .config("spark.ui.enabled", false)
- .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
- .config("spark.driver.maxResultSize", "8G")
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
- .config("spark.sql.catalog.spark_catalog.type", "hadoop")
- .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
- .master("local[*]")
- .getOrCreate();
- }
-
- private void tearDownSpark() {
- spark.stop();
- }
-
- private void initTable() throws NoSuchTableException, ParseException {
- sql(
- "CREATE TABLE %s ( "
- + " `ss_sold_date_sk` INT, "
- + " `ss_sold_time_sk` INT, "
- + " `ss_item_sk` INT, "
- + " `ss_customer_sk` STRING, "
- + " `ss_cdemo_sk` STRING, "
- + " `ss_hdemo_sk` STRING, "
- + " `ss_addr_sk` STRING, "
- + " `ss_store_sk` STRING, "
- + " `ss_promo_sk` STRING, "
- + " `ss_ticket_number` INT, "
- + " `ss_quantity` STRING, "
- + " `ss_wholesale_cost` STRING, "
- + " `ss_list_price` STRING, "
- + " `ss_sales_price` STRING, "
- + " `ss_ext_discount_amt` STRING, "
- + " `ss_ext_sales_price` STRING, "
- + " `ss_ext_wholesale_cost` STRING, "
- + " `ss_ext_list_price` STRING, "
- + " `ss_ext_tax` STRING, "
- + " `ss_coupon_amt` STRING, "
- + " `ss_net_paid` STRING, "
- + " `ss_net_paid_inc_tax` STRING, "
- + " `ss_net_profit` STRING "
- + ")"
- + "USING iceberg "
- + "PARTITIONED BY (%s) "
- + "TBLPROPERTIES ("
- + " '%s' '%b',"
- + " '%s' '%s',"
- + " '%s' '%d')",
- TABLE_NAME,
- PARTITION_COLUMN,
- TableProperties.MANIFEST_MERGE_ENABLED,
- false,
- TableProperties.DELETE_MODE,
- RowLevelOperationMode.MERGE_ON_READ.modeName(),
- TableProperties.FORMAT_VERSION,
- type.equals("dv") ? 3 : 2);
-
- this.table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
- }
-
- private void dropTable() {
- sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
- }
-
- private void initDataAndDeletes() {
- if (type.equals("partition")) {
- initDataAndPartitionScopedDeletes();
- } else if (type.equals("file")) {
- initDataAndFileScopedDeletes();
- } else {
- initDataAndDVs();
- }
- }
-
- private void initDataAndPartitionScopedDeletes() {
- for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
- StructLike partition = TestHelpers.Row.of(partitionOrdinal);
-
- RowDelta rowDelta = table.newRowDelta();
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
- DataFile dataFile = generateDataFile(partition, Integer.MIN_VALUE, Integer.MIN_VALUE);
- rowDelta.addRows(dataFile);
- }
-
- // add one data file that would match the sort key predicate
- DataFile sortKeyDataFile = generateDataFile(partition, SORT_KEY_VALUE, SORT_KEY_VALUE);
- rowDelta.addRows(sortKeyDataFile);
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DELETE_FILES_PER_PARTITION; fileOrdinal++) {
- DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, partition);
- rowDelta.addDeletes(deleteFile);
- }
-
- rowDelta.commit();
- }
- }
-
- private void initDataAndFileScopedDeletes() {
- for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
- StructLike partition = TestHelpers.Row.of(partitionOrdinal);
-
- RowDelta rowDelta = table.newRowDelta();
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
- DataFile dataFile = generateDataFile(partition, Integer.MIN_VALUE, Integer.MIN_VALUE);
- DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
- rowDelta.addRows(dataFile);
- rowDelta.addDeletes(deleteFile);
- }
-
- // add one data file that would match the sort key predicate
- DataFile sortKeyDataFile = generateDataFile(partition, SORT_KEY_VALUE, SORT_KEY_VALUE);
- rowDelta.addRows(sortKeyDataFile);
-
- rowDelta.commit();
- }
- }
-
- private void initDataAndDVs() {
- for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
- StructLike partition = TestHelpers.Row.of(partitionOrdinal);
-
- RowDelta rowDelta = table.newRowDelta();
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
- DataFile dataFile = generateDataFile(partition, Integer.MIN_VALUE, Integer.MIN_VALUE);
- DeleteFile dv = FileGenerationUtil.generateDV(table, dataFile);
- rowDelta.addRows(dataFile);
- rowDelta.addDeletes(dv);
- }
-
- // add one data file that would match the sort key predicate
- DataFile sortKeyDataFile = generateDataFile(partition, SORT_KEY_VALUE, SORT_KEY_VALUE);
- rowDelta.addRows(sortKeyDataFile);
-
- rowDelta.commit();
- }
- }
-
- private DataFile generateDataFile(StructLike partition, int sortKeyMin, int sortKeyMax) {
- int sortKeyFieldId = table.schema().findField(SORT_KEY_COLUMN).fieldId();
- ByteBuffer lower = Conversions.toByteBuffer(Types.IntegerType.get(), sortKeyMin);
- Map<Integer, ByteBuffer> lowerBounds = ImmutableMap.of(sortKeyFieldId, lower);
- ByteBuffer upper = Conversions.toByteBuffer(Types.IntegerType.get(), sortKeyMax);
- Map<Integer, ByteBuffer> upperBounds = ImmutableMap.of(sortKeyFieldId, upper);
- return FileGenerationUtil.generateDataFile(table, partition, lowerBounds, upperBounds);
- }
-
- private String newWarehouseDir() {
- return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
- }
-
- private List<ScanTask> planFilesWithoutColumnStats(BatchScan scan, Expression predicate) {
- return planFiles(scan, predicate, false);
- }
-
- private List<ScanTask> planFilesWithColumnStats(BatchScan scan, Expression predicate) {
- return planFiles(scan, predicate, true);
- }
-
- private List<ScanTask> planFiles(BatchScan scan, Expression predicate, boolean withColumnStats) {
- table.refresh();
-
- BatchScan configuredScan = scan.filter(predicate);
-
- if (withColumnStats) {
- configuredScan = scan.includeColumnStats();
- }
-
- try (CloseableIterable<ScanTask> fileTasks = configuredScan.planFiles()) {
- return Lists.newArrayList(fileTasks);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- private BatchScan newDistributedScan(PlanningMode dataMode, PlanningMode deleteMode) {
- table
- .updateProperties()
- .set(TableProperties.DATA_PLANNING_MODE, dataMode.modeName())
- .set(TableProperties.DELETE_PLANNING_MODE, deleteMode.modeName())
- .commit();
- SparkReadConf readConf = new SparkReadConf(spark, table, ImmutableMap.of());
- return new SparkDistributedDataScan(spark, table, readConf);
- }
-
- @FormatMethod
- private void sql(@FormatString String query, Object... args) {
- spark.sql(String.format(query, args));
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/TaskGroupPlanningBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/TaskGroupPlanningBenchmark.java
deleted file mode 100644
index 7c2def2378..0000000000
--- a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/TaskGroupPlanningBenchmark.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark;
-
-import com.google.errorprone.annotations.FormatMethod;
-import com.google.errorprone.annotations.FormatString;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.DeleteFile;
-import org.apache.iceberg.FileGenerationUtil;
-import org.apache.iceberg.FileScanTask;
-import org.apache.iceberg.Partitioning;
-import org.apache.iceberg.RowDelta;
-import org.apache.iceberg.RowLevelOperationMode;
-import org.apache.iceberg.ScanTaskGroup;
-import org.apache.iceberg.StructLike;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.TestHelpers;
-import org.apache.iceberg.io.CloseableIterable;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
-import org.apache.iceberg.util.TableScanUtil;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.catalyst.parser.ParseException;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Timeout;
-import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
-
-/**
- * A benchmark that evaluates the task group planning performance.
- *
- * <p>To run this benchmark for spark-4.0: <code>
- * ./gradlew -DsparkVersions=4.0 :iceberg-spark:iceberg-spark-extensions-4.0_2.13:jmh
- * -PjmhIncludeRegex=TaskGroupPlanningBenchmark
- * -PjmhOutputPath=benchmark/iceberg-task-group-planning-benchmark.txt
- * </code>
- */
-@Fork(1)
-@State(Scope.Benchmark)
-@Warmup(iterations = 3)
-@Measurement(iterations = 5)
-@Timeout(time = 30, timeUnit = TimeUnit.MINUTES)
-@BenchmarkMode(Mode.SingleShotTime)
-public class TaskGroupPlanningBenchmark {
-
- private static final String TABLE_NAME = "test_table";
- private static final String PARTITION_COLUMN = "ss_ticket_number";
-
- private static final int NUM_PARTITIONS = 150;
- private static final int NUM_DATA_FILES_PER_PARTITION = 50_000;
- private static final int NUM_DELETE_FILES_PER_PARTITION = 25;
-
- private final Configuration hadoopConf = new Configuration();
- private SparkSession spark;
- private Table table;
-
- private List<FileScanTask> fileTasks;
-
- @Setup
- public void setupBenchmark() throws NoSuchTableException, ParseException {
- setupSpark();
- initTable();
- initDataAndDeletes();
- loadFileTasks();
- }
-
- @TearDown
- public void tearDownBenchmark() {
- dropTable();
- tearDownSpark();
- }
-
- @Benchmark
- @Threads(1)
- public void planTaskGroups(Blackhole blackhole) {
- SparkReadConf readConf = new SparkReadConf(spark, table, ImmutableMap.of());
- List<ScanTaskGroup<FileScanTask>> taskGroups =
- TableScanUtil.planTaskGroups(
- fileTasks,
- readConf.splitSize(),
- readConf.splitLookback(),
- readConf.splitOpenFileCost());
-
- long rowsCount = 0L;
- for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
- rowsCount += taskGroup.estimatedRowsCount();
- }
- blackhole.consume(rowsCount);
-
- long filesCount = 0L;
- for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
- filesCount += taskGroup.filesCount();
- }
- blackhole.consume(filesCount);
-
- long sizeBytes = 0L;
- for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
- sizeBytes += taskGroup.sizeBytes();
- }
- blackhole.consume(sizeBytes);
- }
-
- @Benchmark
- @Threads(1)
- public void planTaskGroupsWithGrouping(Blackhole blackhole) {
- SparkReadConf readConf = new SparkReadConf(spark, table, ImmutableMap.of());
-
- List<ScanTaskGroup<FileScanTask>> taskGroups =
- TableScanUtil.planTaskGroups(
- fileTasks,
- readConf.splitSize(),
- readConf.splitLookback(),
- readConf.splitOpenFileCost(),
- Partitioning.groupingKeyType(table.schema(), table.specs().values()));
-
- long rowsCount = 0L;
- for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
- rowsCount += taskGroup.estimatedRowsCount();
- }
- blackhole.consume(rowsCount);
-
- long filesCount = 0L;
- for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
- filesCount += taskGroup.filesCount();
- }
- blackhole.consume(filesCount);
-
- long sizeBytes = 0L;
- for (ScanTaskGroup<FileScanTask> taskGroup : taskGroups) {
- sizeBytes += taskGroup.sizeBytes();
- }
- blackhole.consume(sizeBytes);
- }
-
- private void loadFileTasks() {
- table.refresh();
-
- try (CloseableIterable<FileScanTask> fileTasksIterable = table.newScan().planFiles()) {
- this.fileTasks = Lists.newArrayList(fileTasksIterable);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- private void initDataAndDeletes() {
- for (int partitionOrdinal = 0; partitionOrdinal < NUM_PARTITIONS; partitionOrdinal++) {
- StructLike partition = TestHelpers.Row.of(partitionOrdinal);
-
- RowDelta rowDelta = table.newRowDelta();
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DATA_FILES_PER_PARTITION; fileOrdinal++) {
- DataFile dataFile = FileGenerationUtil.generateDataFile(table, partition);
- rowDelta.addRows(dataFile);
- }
-
- for (int fileOrdinal = 0; fileOrdinal < NUM_DELETE_FILES_PER_PARTITION; fileOrdinal++) {
- DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, partition);
- rowDelta.addDeletes(deleteFile);
- }
-
- rowDelta.commit();
- }
- }
-
- private void setupSpark() {
- this.spark =
- SparkSession.builder()
- .config("spark.ui.enabled", false)
- .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
- .config("spark.sql.catalog.spark_catalog.type", "hadoop")
- .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
- .master("local[*]")
- .getOrCreate();
- }
-
- private void tearDownSpark() {
- spark.stop();
- }
-
- private void initTable() throws NoSuchTableException, ParseException {
- sql(
- "CREATE TABLE %s ( "
- + " `ss_sold_date_sk` INT, "
- + " `ss_sold_time_sk` INT, "
- + " `ss_item_sk` INT, "
- + " `ss_customer_sk` STRING, "
- + " `ss_cdemo_sk` STRING, "
- + " `ss_hdemo_sk` STRING, "
- + " `ss_addr_sk` STRING, "
- + " `ss_store_sk` STRING, "
- + " `ss_promo_sk` STRING, "
- + " `ss_ticket_number` INT, "
- + " `ss_quantity` STRING, "
- + " `ss_wholesale_cost` STRING, "
- + " `ss_list_price` STRING, "
- + " `ss_sales_price` STRING, "
- + " `ss_ext_discount_amt` STRING, "
- + " `ss_ext_sales_price` STRING, "
- + " `ss_ext_wholesale_cost` STRING, "
- + " `ss_ext_list_price` STRING, "
- + " `ss_ext_tax` STRING, "
- + " `ss_coupon_amt` STRING, "
- + " `ss_net_paid` STRING, "
- + " `ss_net_paid_inc_tax` STRING, "
- + " `ss_net_profit` STRING "
- + ")"
- + "USING iceberg "
- + "PARTITIONED BY (%s) "
- + "TBLPROPERTIES ("
- + " '%s' '%b',"
- + " '%s' '%s',"
- + " '%s' '%d')",
- TABLE_NAME,
- PARTITION_COLUMN,
- TableProperties.MANIFEST_MERGE_ENABLED,
- false,
- TableProperties.DELETE_MODE,
- RowLevelOperationMode.MERGE_ON_READ.modeName(),
- TableProperties.FORMAT_VERSION,
- 2);
-
- this.table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
- }
-
- private void dropTable() {
- sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
- }
-
- private String newWarehouseDir() {
- return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
- }
-
- @FormatMethod
- private void sql(@FormatString String query, Object... args) {
- spark.sql(String.format(query, args));
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/UpdateProjectionBenchmark.java b/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/UpdateProjectionBenchmark.java
deleted file mode 100644
index d917eae5eb..0000000000
--- a/spark/v4.0/spark-extensions/src/jmh/java/org/apache/iceberg/spark/UpdateProjectionBenchmark.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark;
-
-import static org.apache.spark.sql.functions.current_date;
-import static org.apache.spark.sql.functions.date_add;
-import static org.apache.spark.sql.functions.expr;
-
-import com.google.errorprone.annotations.FormatMethod;
-import com.google.errorprone.annotations.FormatString;
-import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.DistributionMode;
-import org.apache.iceberg.RowLevelOperationMode;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.catalyst.parser.ParseException;
-import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.sql.types.StructType;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Threads;
-import org.openjdk.jmh.annotations.Warmup;
-
-@Fork(1)
-@State(Scope.Benchmark)
-@Warmup(iterations = 3)
-@Measurement(iterations = 5)
-@BenchmarkMode(Mode.SingleShotTime)
-public class UpdateProjectionBenchmark {
-
- private static final String TABLE_NAME = "test_table";
- private static final int NUM_FILES = 5;
- private static final int NUM_ROWS_PER_FILE = 1_000_000;
-
- private final Configuration hadoopConf = new Configuration();
- private SparkSession spark;
- private long originalSnapshotId;
-
- @Setup
- public void setupBenchmark() throws NoSuchTableException, ParseException {
- setupSpark();
- initTable();
- appendData();
-
- Table table = Spark3Util.loadIcebergTable(spark, TABLE_NAME);
- this.originalSnapshotId = table.currentSnapshot().snapshotId();
- }
-
- @TearDown
- public void tearDownBenchmark() {
- tearDownSpark();
- dropTable();
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteUpdate10Percent() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.1);
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteUpdate30Percent() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.3);
- }
-
- @Benchmark
- @Threads(1)
- public void copyOnWriteUpdate75Percent() {
- runBenchmark(RowLevelOperationMode.COPY_ON_WRITE, 0.75);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnRead10Percent() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.1);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadUpdate30Percent() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.3);
- }
-
- @Benchmark
- @Threads(1)
- public void mergeOnReadUpdate75Percent() {
- runBenchmark(RowLevelOperationMode.MERGE_ON_READ, 0.75);
- }
-
- private void runBenchmark(RowLevelOperationMode mode, double updatePercentage) {
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')",
- TABLE_NAME, TableProperties.UPDATE_MODE, mode.modeName());
-
- int mod = (int) (NUM_ROWS_PER_FILE / (NUM_ROWS_PER_FILE * updatePercentage));
-
- sql(
- "UPDATE %s "
- + "SET intCol = intCol + 10, dateCol = date_add(dateCol, 1) "
- + "WHERE mod(id, %d) = 0",
- TABLE_NAME, mod);
-
- sql(
- "CALL system.rollback_to_snapshot(table => '%s', snapshot_id => %dL)",
- TABLE_NAME, originalSnapshotId);
- }
-
- private void setupSpark() {
- this.spark =
- SparkSession.builder()
- .config("spark.ui.enabled", false)
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName())
- .config("spark.sql.catalog.spark_catalog.type", "hadoop")
- .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir())
- .config(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key(), "false")
- .config(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false")
- .config(SQLConf.SHUFFLE_PARTITIONS().key(), "2")
- .master("local")
- .getOrCreate();
- }
-
- private void tearDownSpark() {
- spark.stop();
- }
-
- private void initTable() {
- sql(
- "CREATE TABLE %s ( "
- + " id LONG, intCol INT, floatCol FLOAT, doubleCol DOUBLE, "
- + " decimalCol DECIMAL(20, 5), dateCol DATE, timestampCol TIMESTAMP, "
- + " stringCol STRING)"
- + "USING iceberg "
- + "TBLPROPERTIES ("
- + " '%s' '%s',"
- + " '%s' '%d',"
- + " '%s' '%d')",
- TABLE_NAME,
- TableProperties.UPDATE_DISTRIBUTION_MODE,
- DistributionMode.NONE.modeName(),
- TableProperties.SPLIT_OPEN_FILE_COST,
- Integer.MAX_VALUE,
- TableProperties.FORMAT_VERSION,
- 2);
-
- sql("ALTER TABLE %s WRITE ORDERED BY id", TABLE_NAME);
- }
-
- private void dropTable() {
- sql("DROP TABLE IF EXISTS %s PURGE", TABLE_NAME);
- }
-
- private void appendData() throws NoSuchTableException {
- for (int fileNum = 1; fileNum <= NUM_FILES; fileNum++) {
- Dataset<Row> inputDF =
- spark
- .range(NUM_ROWS_PER_FILE)
- .withColumn("intCol", expr("CAST(id AS INT)"))
- .withColumn("floatCol", expr("CAST(id AS FLOAT)"))
- .withColumn("doubleCol", expr("CAST(id AS DOUBLE)"))
- .withColumn("decimalCol", expr("CAST(id AS DECIMAL(20, 5))"))
- .withColumn("dateCol", date_add(current_date(), fileNum))
- .withColumn("timestampCol", expr("TO_TIMESTAMP(dateCol)"))
- .withColumn("stringCol", expr("CAST(dateCol AS STRING)"));
- appendAsFile(inputDF);
- }
- }
-
- private void appendAsFile(Dataset<Row> df) throws NoSuchTableException {
- // ensure the schema is precise (including nullability)
- StructType sparkSchema = spark.table(TABLE_NAME).schema();
- spark.createDataFrame(df.rdd(), sparkSchema).coalesce(1).writeTo(TABLE_NAME).append();
- }
-
- private String newWarehouseDir() {
- return hadoopConf.get("hadoop.tmp.dir") + UUID.randomUUID();
- }
-
- @FormatMethod
- private void sql(@FormatString String query, Object... args) {
- spark.sql(String.format(query, args));
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 b/spark/v4.0/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
deleted file mode 100644
index b962699d9b..0000000000
--- a/spark/v4.0/spark-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * This file is an adaptation of Presto's and Spark's grammar files.
- */
-
-grammar IcebergSqlExtensions;
-
-@lexer::members {
- /**
- * Verify whether current token is a valid decimal token (which contains dot).
- * Returns true if the character that follows the token is not a digit or letter or underscore.
- *
- * For example:
- * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'.
- * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'.
- * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'.
- * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed
- * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+'
- * which is not a digit or letter or underscore.
- */
- public boolean isValidDecimal() {
- int nextChar = _input.LA(1);
- if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' ||
- nextChar == '_') {
- return false;
- } else {
- return true;
- }
- }
-
- /**
- * This method will be called when we see '/*' and try to match it as a bracketed comment.
- * If the next character is '+', it should be parsed as hint later, and we cannot match
- * it as a bracketed comment.
- *
- * Returns true if the next character is '+'.
- */
- public boolean isHint() {
- int nextChar = _input.LA(1);
- if (nextChar == '+') {
- return true;
- } else {
- return false;
- }
- }
-}
-
-singleStatement
- : statement EOF
- ;
-
-statement
- : CALL multipartIdentifier '(' (callArgument (',' callArgument)*)? ')' #call
- | ALTER TABLE multipartIdentifier ADD PARTITION FIELD transform (AS name=identifier)? #addPartitionField
- | ALTER TABLE multipartIdentifier DROP PARTITION FIELD transform #dropPartitionField
- | ALTER TABLE multipartIdentifier REPLACE PARTITION FIELD transform WITH transform (AS name=identifier)? #replacePartitionField
- | ALTER TABLE multipartIdentifier WRITE writeSpec #setWriteDistributionAndOrdering
- | ALTER TABLE multipartIdentifier SET IDENTIFIER_KW FIELDS fieldList #setIdentifierFields
- | ALTER TABLE multipartIdentifier DROP IDENTIFIER_KW FIELDS fieldList #dropIdentifierFields
- | ALTER TABLE multipartIdentifier createReplaceBranchClause #createOrReplaceBranch
- | ALTER TABLE multipartIdentifier createReplaceTagClause #createOrReplaceTag
- | ALTER TABLE multipartIdentifier DROP BRANCH (IF EXISTS)? identifier #dropBranch
- | ALTER TABLE multipartIdentifier DROP TAG (IF EXISTS)? identifier #dropTag
- ;
-
-createReplaceTagClause
- : (CREATE OR)? REPLACE TAG identifier tagOptions
- | CREATE TAG (IF NOT EXISTS)? identifier tagOptions
- ;
-
-createReplaceBranchClause
- : (CREATE OR)? REPLACE BRANCH identifier branchOptions
- | CREATE BRANCH (IF NOT EXISTS)? identifier branchOptions
- ;
-
-tagOptions
- : (AS OF VERSION snapshotId)? (refRetain)?
- ;
-
-branchOptions
- : (AS OF VERSION snapshotId)? (refRetain)? (snapshotRetention)?
- ;
-
-snapshotRetention
- : WITH SNAPSHOT RETENTION minSnapshotsToKeep
- | WITH SNAPSHOT RETENTION maxSnapshotAge
- | WITH SNAPSHOT RETENTION minSnapshotsToKeep maxSnapshotAge
- ;
-
-refRetain
- : RETAIN number timeUnit
- ;
-
-maxSnapshotAge
- : number timeUnit
- ;
-
-minSnapshotsToKeep
- : number SNAPSHOTS
- ;
-
-writeSpec
- : (writeDistributionSpec | writeOrderingSpec)*
- ;
-
-writeDistributionSpec
- : DISTRIBUTED BY PARTITION
- ;
-
-writeOrderingSpec
- : LOCALLY? ORDERED BY order
- | UNORDERED
- ;
-
-callArgument
- : expression #positionalArgument
- | identifier '=>' expression #namedArgument
- ;
-
-singleOrder
- : order EOF
- ;
-
-order
- : fields+=orderField (',' fields+=orderField)*
- | '(' fields+=orderField (',' fields+=orderField)* ')'
- ;
-
-orderField
- : transform direction=(ASC | DESC)? (NULLS nullOrder=(FIRST | LAST))?
- ;
-
-transform
- : multipartIdentifier #identityTransform
- | transformName=identifier
- '(' arguments+=transformArgument (',' arguments+=transformArgument)* ')' #applyTransform
- ;
-
-transformArgument
- : multipartIdentifier
- | constant
- ;
-
-expression
- : constant
- | stringMap
- | stringArray
- ;
-
-constant
- : number #numericLiteral
- | booleanValue #booleanLiteral
- | STRING+ #stringLiteral
- | identifier STRING #typeConstructor
- ;
-
-stringMap
- : MAP '(' constant (',' constant)* ')'
- ;
-
-stringArray
- : ARRAY '(' constant (',' constant)* ')'
- ;
-
-booleanValue
- : TRUE | FALSE
- ;
-
-number
- : MINUS? EXPONENT_VALUE #exponentLiteral
- | MINUS? DECIMAL_VALUE #decimalLiteral
- | MINUS? INTEGER_VALUE #integerLiteral
- | MINUS? BIGINT_LITERAL #bigIntLiteral
- | MINUS? SMALLINT_LITERAL #smallIntLiteral
- | MINUS? TINYINT_LITERAL #tinyIntLiteral
- | MINUS? DOUBLE_LITERAL #doubleLiteral
- | MINUS? FLOAT_LITERAL #floatLiteral
- | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral
- ;
-
-multipartIdentifier
- : parts+=identifier ('.' parts+=identifier)*
- ;
-
-identifier
- : IDENTIFIER #unquotedIdentifier
- | quotedIdentifier #quotedIdentifierAlternative
- | nonReserved #unquotedIdentifier
- ;
-
-quotedIdentifier
- : BACKQUOTED_IDENTIFIER
- ;
-
-fieldList
- : fields+=multipartIdentifier (',' fields+=multipartIdentifier)*
- ;
-
-nonReserved
- : ADD | ALTER | AS | ASC | BRANCH | BY | CALL | CREATE | DAYS | DESC | DROP | EXISTS | FIELD | FIRST | HOURS | IF | LAST | NOT | NULLS | OF | OR | ORDERED | PARTITION | TABLE | WRITE
- | DISTRIBUTED | LOCALLY | MINUTES | MONTHS | UNORDERED | REPLACE | RETAIN | VERSION | WITH | IDENTIFIER_KW | FIELDS | SET | SNAPSHOT | SNAPSHOTS
- | TAG | TRUE | FALSE
- | MAP
- ;
-
-snapshotId
- : number
- ;
-
-numSnapshots
- : number
- ;
-
-timeUnit
- : DAYS
- | HOURS
- | MINUTES
- ;
-
-ADD: 'ADD';
-ALTER: 'ALTER';
-AS: 'AS';
-ASC: 'ASC';
-BRANCH: 'BRANCH';
-BY: 'BY';
-CALL: 'CALL';
-DAYS: 'DAYS';
-DESC: 'DESC';
-DISTRIBUTED: 'DISTRIBUTED';
-DROP: 'DROP';
-EXISTS: 'EXISTS';
-FIELD: 'FIELD';
-FIELDS: 'FIELDS';
-FIRST: 'FIRST';
-HOURS: 'HOURS';
-IF : 'IF';
-LAST: 'LAST';
-LOCALLY: 'LOCALLY';
-MINUTES: 'MINUTES';
-MONTHS: 'MONTHS';
-CREATE: 'CREATE';
-NOT: 'NOT';
-NULLS: 'NULLS';
-OF: 'OF';
-OR: 'OR';
-ORDERED: 'ORDERED';
-PARTITION: 'PARTITION';
-REPLACE: 'REPLACE';
-RETAIN: 'RETAIN';
-RETENTION: 'RETENTION';
-IDENTIFIER_KW: 'IDENTIFIER';
-SET: 'SET';
-SNAPSHOT: 'SNAPSHOT';
-SNAPSHOTS: 'SNAPSHOTS';
-TABLE: 'TABLE';
-TAG: 'TAG';
-UNORDERED: 'UNORDERED';
-VERSION: 'VERSION';
-WITH: 'WITH';
-WRITE: 'WRITE';
-
-TRUE: 'TRUE';
-FALSE: 'FALSE';
-
-MAP: 'MAP';
-ARRAY: 'ARRAY';
-
-PLUS: '+';
-MINUS: '-';
-
-STRING
- : '\'' ( ~('\''|'\\') | ('\\' .) )* '\''
- | '"' ( ~('"'|'\\') | ('\\' .) )* '"'
- ;
-
-BIGINT_LITERAL
- : DIGIT+ 'L'
- ;
-
-SMALLINT_LITERAL
- : DIGIT+ 'S'
- ;
-
-TINYINT_LITERAL
- : DIGIT+ 'Y'
- ;
-
-INTEGER_VALUE
- : DIGIT+
- ;
-
-EXPONENT_VALUE
- : DIGIT+ EXPONENT
- | DECIMAL_DIGITS EXPONENT {isValidDecimal()}?
- ;
-
-DECIMAL_VALUE
- : DECIMAL_DIGITS {isValidDecimal()}?
- ;
-
-FLOAT_LITERAL
- : DIGIT+ EXPONENT? 'F'
- | DECIMAL_DIGITS EXPONENT? 'F' {isValidDecimal()}?
- ;
-
-DOUBLE_LITERAL
- : DIGIT+ EXPONENT? 'D'
- | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}?
- ;
-
-BIGDECIMAL_LITERAL
- : DIGIT+ EXPONENT? 'BD'
- | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}?
- ;
-
-IDENTIFIER
- : (LETTER | DIGIT | '_')+
- ;
-
-BACKQUOTED_IDENTIFIER
- : '`' ( ~'`' | '``' )* '`'
- ;
-
-fragment DECIMAL_DIGITS
- : DIGIT+ '.' DIGIT*
- | '.' DIGIT+
- ;
-
-fragment EXPONENT
- : 'E' [+-]? DIGIT+
- ;
-
-fragment DIGIT
- : [0-9]
- ;
-
-fragment LETTER
- : [A-Z]
- ;
-
-SIMPLE_COMMENT
- : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
- ;
-
-BRACKETED_COMMENT
- : '/*' {!isHint()}? (BRACKETED_COMMENT|.)*? '*/' -> channel(HIDDEN)
- ;
-
-WS
- : [ \r\n\t]+ -> channel(HIDDEN)
- ;
-
-// Catch-all for anything we can't recognize.
-// We use this to be able to ignore and recover all the text
-// when splitting statements with DelimiterLexer
-UNRECOGNIZED
- : .
- ;
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
deleted file mode 100644
index 3fca29c294..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iceberg.spark.extensions
-
-import org.apache.spark.sql.SparkSessionExtensions
-import org.apache.spark.sql.catalyst.analysis.CheckViews
-import org.apache.spark.sql.catalyst.analysis.ProcedureArgumentCoercion
-import org.apache.spark.sql.catalyst.analysis.ResolveProcedures
-import org.apache.spark.sql.catalyst.analysis.ResolveViews
-import org.apache.spark.sql.catalyst.optimizer.ReplaceStaticInvoke
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser
-import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy
-
-class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
-
- override def apply(extensions: SparkSessionExtensions): Unit = {
- // parser extensions
- extensions.injectParser { case (_, parser) => new IcebergSparkSqlExtensionsParser(parser) }
-
- // analyzer extensions
- extensions.injectResolutionRule { spark => ResolveProcedures(spark) }
- extensions.injectResolutionRule { spark => ResolveViews(spark) }
- extensions.injectResolutionRule { _ => ProcedureArgumentCoercion }
- extensions.injectCheckRule(_ => CheckViews)
-
- // optimizer extensions
- extensions.injectOptimizerRule { _ => ReplaceStaticInvoke }
-
- // planner extensions
- extensions.injectPlannerStrategy { spark => ExtendedDataSourceV2Strategy(spark) }
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
deleted file mode 100644
index 109aa5360e..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.plans.logical.AlterViewAs
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias
-import org.apache.spark.sql.catalyst.plans.logical.View
-import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
-import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.util.SchemaUtils
-
-object CheckViews extends (LogicalPlan => Unit) {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override def apply(plan: LogicalPlan): Unit = {
- plan foreach {
- case CreateIcebergView(resolvedIdent@ResolvedIdentifier(_: ViewCatalog, _), _, query, columnAliases, _,
- _, _, _, _, replace, _) =>
- verifyColumnCount(resolvedIdent, columnAliases, query)
- SchemaUtils.checkColumnNameDuplication(query.schema.fieldNames, SQLConf.get.resolver)
- if (replace) {
- val viewIdent: Seq[String] = resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
- checkCyclicViewReference(viewIdent, query, Seq(viewIdent))
- }
-
- case AlterViewAs(ResolvedV2View(_, _), _, _) =>
- throw new IcebergAnalysisException(
- "ALTER VIEW <viewName> AS is not supported. Use CREATE OR REPLACE VIEW instead")
-
- case _ => // OK
- }
- }
-
- private def verifyColumnCount(ident: ResolvedIdentifier, columns: Seq[String], query: LogicalPlan): Unit = {
- if (columns.nonEmpty) {
- if (columns.length > query.output.length) {
- throw new AnalysisException(
- errorClass = "CREATE_VIEW_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS",
- messageParameters = Map(
- "viewName" -> String.format("%s.%s", ident.catalog.name(), ident.identifier),
- "viewColumns" -> columns.mkString(", "),
- "dataColumns" -> query.output.map(c => c.name).mkString(", ")))
- } else if (columns.length < query.output.length) {
- throw new AnalysisException(
- errorClass = "CREATE_VIEW_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS",
- messageParameters = Map(
- "viewName" -> String.format("%s.%s", ident.catalog.name(), ident.identifier),
- "viewColumns" -> columns.mkString(", "),
- "dataColumns" -> query.output.map(c => c.name).mkString(", ")))
- }
- }
- }
-
- private def checkCyclicViewReference(
- viewIdent: Seq[String],
- plan: LogicalPlan,
- cyclePath: Seq[Seq[String]]): Unit = {
- plan match {
- case sub@SubqueryAlias(_, Project(_, _)) =>
- val currentViewIdent: Seq[String] = sub.identifier.qualifier :+ sub.identifier.name
- checkIfRecursiveView(viewIdent, currentViewIdent, cyclePath, sub.children)
- case v1View: View =>
- val currentViewIdent: Seq[String] = v1View.desc.identifier.nameParts
- checkIfRecursiveView(viewIdent, currentViewIdent, cyclePath, v1View.children)
- case _ =>
- plan.children.foreach(child => checkCyclicViewReference(viewIdent, child, cyclePath))
- }
-
- plan.expressions.flatMap(_.flatMap {
- case e: SubqueryExpression =>
- checkCyclicViewReference(viewIdent, e.plan, cyclePath)
- None
- case _ => None
- })
- }
-
- private def checkIfRecursiveView(
- viewIdent: Seq[String],
- currentViewIdent: Seq[String],
- cyclePath: Seq[Seq[String]],
- children: Seq[LogicalPlan]
- ): Unit = {
- val newCyclePath = cyclePath :+ currentViewIdent
- if (currentViewIdent == viewIdent) {
- throw new IcebergAnalysisException(String.format("Recursive cycle in view detected: %s (cycle: %s)",
- viewIdent.asIdentifier, newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
- } else {
- children.foreach { c =>
- checkCyclicViewReference(viewIdent, c, newCyclePath)
- }
- }
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
deleted file mode 100644
index 01dbd69526..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.catalyst.expressions.Cast
-import org.apache.spark.sql.catalyst.plans.logical.IcebergCall
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.rules.Rule
-
-object ProcedureArgumentCoercion extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case c @ IcebergCall(procedure, args) if c.resolved =>
- val params = procedure.parameters
-
- val newArgs = args.zipWithIndex.map { case (arg, index) =>
- val param = params(index)
- val paramType = param.dataType
- val argType = arg.dataType
-
- if (paramType != argType && !Cast.canUpCast(argType, paramType)) {
- throw new IcebergAnalysisException(
- s"Wrong arg type for ${param.name}: cannot cast $argType to $paramType")
- }
-
- if (paramType != argType) {
- Cast(arg, paramType)
- } else {
- arg
- }
- }
-
- if (newArgs != args) {
- c.copy(args = newArgs)
- } else {
- c
- }
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
deleted file mode 100644
index 2d02d4ce76..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import java.util.Locale
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.plans.logical.CallArgument
-import org.apache.spark.sql.catalyst.plans.logical.CallStatement
-import org.apache.spark.sql.catalyst.plans.logical.IcebergCall
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.NamedArgument
-import org.apache.spark.sql.catalyst.plans.logical.PositionalArgument
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.connector.catalog.CatalogPlugin
-import org.apache.spark.sql.connector.catalog.LookupCatalog
-import org.apache.spark.sql.connector.iceberg.catalog.ProcedureCatalog
-import org.apache.spark.sql.connector.iceberg.catalog.ProcedureParameter
-import scala.collection.Seq
-
-case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog {
-
- protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case CallStatement(CatalogAndIdentifier(catalog, ident), args) =>
- val procedure = catalog.asProcedureCatalog.loadProcedure(ident)
-
- val params = procedure.parameters
- val normalizedParams = normalizeParams(params)
- validateParams(normalizedParams)
-
- val normalizedArgs = normalizeArgs(args)
- IcebergCall(procedure, args = buildArgExprs(normalizedParams, normalizedArgs).toSeq)
- }
-
- private def validateParams(params: Seq[ProcedureParameter]): Unit = {
- // should not be any duplicate param names
- val duplicateParamNames = params.groupBy(_.name).collect {
- case (name, matchingParams) if matchingParams.length > 1 => name
- }
-
- if (duplicateParamNames.nonEmpty) {
- throw new IcebergAnalysisException(s"Duplicate parameter names: ${duplicateParamNames.mkString("[", ",", "]")}")
- }
-
- // optional params should be at the end
- params.sliding(2).foreach {
- case Seq(previousParam, currentParam) if !previousParam.required && currentParam.required =>
- throw new IcebergAnalysisException(
- s"Optional parameters must be after required ones but $currentParam is after $previousParam")
- case _ =>
- }
- }
-
- private def buildArgExprs(
- params: Seq[ProcedureParameter],
- args: Seq[CallArgument]): Seq[Expression] = {
-
- // build a map of declared parameter names to their positions
- val nameToPositionMap = params.map(_.name).zipWithIndex.toMap
-
- // build a map of parameter names to args
- val nameToArgMap = buildNameToArgMap(params, args, nameToPositionMap)
-
- // verify all required parameters are provided
- val missingParamNames = params.filter(_.required).collect {
- case param if !nameToArgMap.contains(param.name) => param.name
- }
-
- if (missingParamNames.nonEmpty) {
- throw new IcebergAnalysisException(s"Missing required parameters: ${missingParamNames.mkString("[", ",", "]")}")
- }
-
- val argExprs = new Array[Expression](params.size)
-
- nameToArgMap.foreach { case (name, arg) =>
- val position = nameToPositionMap(name)
- argExprs(position) = arg.expr
- }
-
- // assign nulls to optional params that were not set
- params.foreach {
- case p if !p.required && !nameToArgMap.contains(p.name) =>
- val position = nameToPositionMap(p.name)
- argExprs(position) = Literal.create(null, p.dataType)
- case _ =>
- }
-
- argExprs
- }
-
- private def buildNameToArgMap(
- params: Seq[ProcedureParameter],
- args: Seq[CallArgument],
- nameToPositionMap: Map[String, Int]): Map[String, CallArgument] = {
-
- val containsNamedArg = args.exists(_.isInstanceOf[NamedArgument])
- val containsPositionalArg = args.exists(_.isInstanceOf[PositionalArgument])
-
- if (containsNamedArg && containsPositionalArg) {
- throw new IcebergAnalysisException("Named and positional arguments cannot be mixed")
- }
-
- if (containsNamedArg) {
- buildNameToArgMapUsingNames(args, nameToPositionMap)
- } else {
- buildNameToArgMapUsingPositions(args, params)
- }
- }
-
- private def buildNameToArgMapUsingNames(
- args: Seq[CallArgument],
- nameToPositionMap: Map[String, Int]): Map[String, CallArgument] = {
-
- val namedArgs = args.asInstanceOf[Seq[NamedArgument]]
-
- val validationErrors = namedArgs.groupBy(_.name).collect {
- case (name, matchingArgs) if matchingArgs.size > 1 => s"Duplicate procedure argument: $name"
- case (name, _) if !nameToPositionMap.contains(name) => s"Unknown argument: $name"
- }
-
- if (validationErrors.nonEmpty) {
- throw new IcebergAnalysisException(s"Could not build name to arg map: ${validationErrors.mkString(", ")}")
- }
-
- namedArgs.map(arg => arg.name -> arg).toMap
- }
-
- private def buildNameToArgMapUsingPositions(
- args: Seq[CallArgument],
- params: Seq[ProcedureParameter]): Map[String, CallArgument] = {
-
- if (args.size > params.size) {
- throw new IcebergAnalysisException("Too many arguments for procedure")
- }
-
- args.zipWithIndex.map { case (arg, position) =>
- val param = params(position)
- param.name -> arg
- }.toMap
- }
-
- private def normalizeParams(params: Seq[ProcedureParameter]): Seq[ProcedureParameter] = {
- params.map {
- case param if param.required =>
- val normalizedName = param.name.toLowerCase(Locale.ROOT)
- ProcedureParameter.required(normalizedName, param.dataType)
- case param =>
- val normalizedName = param.name.toLowerCase(Locale.ROOT)
- ProcedureParameter.optional(normalizedName, param.dataType)
- }
- }
-
- private def normalizeArgs(args: Seq[CallArgument]): Seq[CallArgument] = {
- args.map {
- case a @ NamedArgument(name, _) => a.copy(name = name.toLowerCase(Locale.ROOT))
- case other => other
- }
- }
-
- implicit class CatalogHelper(plugin: CatalogPlugin) {
- def asProcedureCatalog: ProcedureCatalog = plugin match {
- case procedureCatalog: ProcedureCatalog =>
- procedureCatalog
- case _ =>
- throw new IcebergAnalysisException(s"Cannot use catalog ${plugin.name}: not a ProcedureCatalog")
- }
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
deleted file mode 100644
index e69db3fce5..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.analysis.ViewUtil.IcebergViewHelper
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.expressions.UpCast
-import org.apache.spark.sql.catalyst.parser.ParseException
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.Project
-import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias
-import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
-import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.CurrentOrigin
-import org.apache.spark.sql.catalyst.trees.Origin
-import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.connector.catalog.LookupCatalog
-import org.apache.spark.sql.connector.catalog.View
-import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.MetadataBuilder
-
-case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case u@UnresolvedRelation(nameParts, _, _)
- if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
- u
-
- case u@UnresolvedRelation(parts@CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
- .map(createViewRelation(parts, _))
- .getOrElse(u)
-
- case u@UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
- .map(_ => ResolvedV2View(catalog.asViewCatalog, ident))
- .getOrElse(u)
-
- case c@CreateIcebergView(ResolvedIdentifier(_, _), _, query, columnAliases, columnComments, _, _, _, _, _, _)
- if query.resolved && !c.rewritten =>
- val aliased = aliasColumns(query, columnAliases, columnComments)
- c.copy(query = aliased, queryColumnNames = query.schema.fieldNames, rewritten = true)
- }
-
- private def aliasColumns(
- plan: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]]): LogicalPlan = {
- if (columnAliases.isEmpty || columnAliases.length != plan.output.length) {
- plan
- } else {
- val projectList = plan.output.zipWithIndex.map { case (attr, pos) =>
- if (columnComments.apply(pos).isDefined) {
- val meta = new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
- Alias(attr, columnAliases.apply(pos))(explicitMetadata = Some(meta))
- } else {
- Alias(attr, columnAliases.apply(pos))()
- }
- }
- Project(projectList, plan)
- }
- }
-
-
- private def createViewRelation(nameParts: Seq[String], view: View): LogicalPlan = {
- val parsed = parseViewText(nameParts.quoted, view.query)
-
- // Apply any necessary rewrites to preserve correct resolution
- val viewCatalogAndNamespace: Seq[String] = view.currentCatalog +: view.currentNamespace.toSeq
- val rewritten = rewriteIdentifiers(parsed, viewCatalogAndNamespace);
-
- // Apply the field aliases and column comments
- // This logic differs from how Spark handles views in SessionCatalog.fromCatalogTable.
- // This is more strict because it doesn't allow resolution by field name.
- val aliases = view.schema.fields.zipWithIndex.map { case (expected, pos) =>
- val attr = GetColumnByOrdinal(pos, expected.dataType)
- Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata = Some(expected.metadata))
- }
-
- SubqueryAlias(nameParts, Project(aliases, rewritten))
- }
-
- private def parseViewText(name: String, viewText: String): LogicalPlan = {
- val origin = Origin(
- objectType = Some("VIEW"),
- objectName = Some(name)
- )
-
- try {
- CurrentOrigin.withOrigin(origin) {
- spark.sessionState.sqlParser.parseQuery(viewText)
- }
- } catch {
- case _: ParseException =>
- throw QueryCompilationErrors.invalidViewNameError(name)
- }
- }
-
- private def rewriteIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = {
- // Substitute CTEs and Unresolved Ordinals within the view, then rewrite unresolved functions and relations
- qualifyTableIdentifiers(
- qualifyFunctionIdentifiers(
- SubstituteUnresolvedOrdinals.apply(CTESubstitution.apply(plan)),
- catalogAndNamespace),
- catalogAndNamespace)
- }
-
- private def qualifyFunctionIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
- case u@UnresolvedFunction(Seq(name), _, _, _, _, _, _) =>
- if (!isBuiltinFunction(name)) {
- u.copy(nameParts = catalogAndNamespace :+ name)
- } else {
- u
- }
- case u@UnresolvedFunction(parts, _, _, _, _, _, _) if !isCatalog(parts.head) =>
- u.copy(nameParts = catalogAndNamespace.head +: parts)
- }
-
- /**
- * Qualify table identifiers with default catalog and namespace if necessary.
- */
- private def qualifyTableIdentifiers(
- child: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan =
- child transform {
- case u@UnresolvedRelation(Seq(table), _, _) =>
- u.copy(multipartIdentifier = catalogAndNamespace :+ table)
- case u@UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
- u.copy(multipartIdentifier = catalogAndNamespace.head +: parts)
- case other =>
- other.transformExpressions {
- case subquery: SubqueryExpression =>
- subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
- }
- }
-
- private def isCatalog(name: String): Boolean = {
- catalogManager.isCatalogRegistered(name)
- }
-
- private def isBuiltinFunction(name: String): Boolean = {
- catalogManager.v1SessionCatalog.isBuiltinFunction(FunctionIdentifier(name))
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
deleted file mode 100644
index db9ecfcd9e..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.analysis.ViewUtil.IcebergViewHelper
-import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.plans.logical.CreateView
-import org.apache.spark.sql.catalyst.plans.logical.DropView
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.ShowViews
-import org.apache.spark.sql.catalyst.plans.logical.View
-import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
-import org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView
-import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
-import org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.UNRESOLVED_FUNCTION
-import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.connector.catalog.LookupCatalog
-import scala.collection.mutable
-
-/**
- * ResolveSessionCatalog exits early for some v2 View commands,
- * thus they are pre-substituted here and then handled in ResolveViews
- */
-case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
-
- override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
- case DropView(ResolvedIdent(resolved), ifExists) =>
- DropIcebergView(resolved, ifExists)
-
- case CreateView(ResolvedIdent(resolved), userSpecifiedColumns, comment, _, properties,
- Some(queryText), query, allowExisting, replace, _) =>
- val q = CTESubstitution.apply(query)
- verifyTemporaryObjectsDontExist(resolved, q)
- CreateIcebergView(child = resolved,
- queryText = queryText,
- query = q,
- columnAliases = userSpecifiedColumns.map(_._1),
- columnComments = userSpecifiedColumns.map(_._2.orElse(Option.empty)),
- comment = comment,
- properties = properties,
- allowExisting = allowExisting,
- replace = replace)
-
- case view @ ShowViews(CurrentNamespace, pattern, output) =>
- if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) {
- ShowIcebergViews(ResolvedNamespace(catalogManager.currentCatalog, catalogManager.currentNamespace),
- pattern, output)
- } else {
- view
- }
-
- case ShowViews(UnresolvedNamespace(CatalogAndNamespace(catalog, ns), _), pattern, output)
- if ViewUtil.isViewCatalog(catalog) =>
- ShowIcebergViews(ResolvedNamespace(catalog, ns), pattern, output)
-
- // needs to be done here instead of in ResolveViews, so that a V2 view can be resolved before the Analyzer
- // tries to resolve it, which would result in an error, saying that V2 views aren't supported
- case u@UnresolvedView(ResolvedView(resolved), _, _, _) =>
- ViewUtil.loadView(resolved.catalog, resolved.identifier)
- .map(_ => ResolvedV2View(resolved.catalog.asViewCatalog, resolved.identifier))
- .getOrElse(u)
- }
-
- private def isTempView(nameParts: Seq[String]): Boolean = {
- catalogManager.v1SessionCatalog.isTempView(nameParts)
- }
-
- private def isTempFunction(nameParts: Seq[String]): Boolean = {
- if (nameParts.size > 1) {
- return false
- }
- catalogManager.v1SessionCatalog.isTemporaryFunction(nameParts.asFunctionIdentifier)
- }
-
- private object ResolvedIdent {
- def unapply(unresolved: UnresolvedIdentifier): Option[ResolvedIdentifier] = unresolved match {
- case UnresolvedIdentifier(nameParts, true) if isTempView(nameParts) =>
- None
-
- case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _) if ViewUtil.isViewCatalog(catalog) =>
- Some(ResolvedIdentifier(catalog, ident))
-
- case _ =>
- None
- }
- }
-
- /**
- * Permanent views are not allowed to reference temp objects
- */
- private def verifyTemporaryObjectsDontExist(
- identifier: ResolvedIdentifier,
- child: LogicalPlan): Unit = {
- val tempViews = collectTemporaryViews(child)
- if (tempViews.nonEmpty) {
- throw invalidRefToTempObject(identifier, tempViews.map(v => v.quoted).mkString("[", ", ", "]"), "view")
- }
-
- val tempFunctions = collectTemporaryFunctions(child)
- if (tempFunctions.nonEmpty) {
- throw invalidRefToTempObject(identifier, tempFunctions.mkString("[", ", ", "]"), "function")
- }
- }
-
- private def invalidRefToTempObject(ident: ResolvedIdentifier, tempObjectNames: String, tempObjectType: String) = {
- new IcebergAnalysisException(String.format("Cannot create view %s.%s that references temporary %s: %s",
- ident.catalog.name(), ident.identifier, tempObjectType, tempObjectNames))
- }
-
- /**
- * Collect all temporary views and return the identifiers separately
- */
- private def collectTemporaryViews(child: LogicalPlan): Seq[Seq[String]] = {
- def collectTempViews(child: LogicalPlan): Seq[Seq[String]] = {
- child.flatMap {
- case unresolved: UnresolvedRelation if isTempView(unresolved.multipartIdentifier) =>
- Seq(unresolved.multipartIdentifier)
- case view: View if view.isTempView => Seq(view.desc.identifier.nameParts)
- case plan => plan.expressions.flatMap(_.flatMap {
- case e: SubqueryExpression => collectTempViews(e.plan)
- case _ => Seq.empty
- })
- }.distinct
- }
-
- collectTempViews(child)
- }
-
- private object ResolvedView {
- def unapply(identifier: Seq[String]): Option[ResolvedV2View] = identifier match {
- case nameParts if isTempView(nameParts) =>
- None
-
- case CatalogAndIdentifier(catalog, ident) if ViewUtil.isViewCatalog(catalog) =>
- ViewUtil.loadView(catalog, ident).flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
-
- case _ =>
- None
- }
- }
-
- /**
- * Collect the names of all temporary functions.
- */
- private def collectTemporaryFunctions(child: LogicalPlan): Seq[String] = {
- val tempFunctions = new mutable.HashSet[String]()
- child.resolveExpressionsWithPruning(_.containsAnyPattern(UNRESOLVED_FUNCTION)) {
- case f @ UnresolvedFunction(nameParts, _, _, _, _, _, _) if isTempFunction(nameParts) =>
- tempFunctions += nameParts.head
- f
- case e: SubqueryExpression =>
- tempFunctions ++= collectTemporaryFunctions(e.plan)
- e
- }
- tempFunctions.toSeq
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
deleted file mode 100644
index d46f10b7f5..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.analysis
-
-import org.apache.spark.sql.connector.catalog.CatalogPlugin
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.View
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.errors.QueryCompilationErrors
-
-object ViewUtil {
- def loadView(catalog: CatalogPlugin, ident: Identifier): Option[View] = catalog match {
- case viewCatalog: ViewCatalog =>
- try {
- Option(viewCatalog.loadView(ident))
- } catch {
- case _: NoSuchViewException => None
- }
- case _ => None
- }
-
- def isViewCatalog(catalog: CatalogPlugin): Boolean = {
- catalog.isInstanceOf[ViewCatalog]
- }
-
- implicit class IcebergViewHelper(plugin: CatalogPlugin) {
- def asViewCatalog: ViewCatalog = plugin match {
- case viewCatalog: ViewCatalog =>
- viewCatalog
- case _ =>
- throw QueryCompilationErrors.missingCatalogAbilityError(plugin, "views")
- }
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
deleted file mode 100644
index 5f78a0b279..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.spark.sql.catalyst.optimizer
-
-import org.apache.iceberg.spark.functions.SparkFunctions
-import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression
-import org.apache.spark.sql.catalyst.expressions.BinaryComparison
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.In
-import org.apache.spark.sql.catalyst.expressions.InSet
-import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
-import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.plans.logical.Join
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.ReplaceData
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.trees.TreePattern.BINARY_COMPARISON
-import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND
-import org.apache.spark.sql.catalyst.trees.TreePattern.FILTER
-import org.apache.spark.sql.catalyst.trees.TreePattern.IN
-import org.apache.spark.sql.catalyst.trees.TreePattern.INSET
-import org.apache.spark.sql.catalyst.trees.TreePattern.JOIN
-import org.apache.spark.sql.connector.catalog.functions.ScalarFunction
-import org.apache.spark.sql.types.StructField
-import org.apache.spark.sql.types.StructType
-
-/**
- * Spark analyzes the Iceberg system function to {@link StaticInvoke} which could not be pushed
- * down to datasource. This rule will replace {@link StaticInvoke} to
- * {@link ApplyFunctionExpression} for Iceberg system function in a filter condition.
- */
-object ReplaceStaticInvoke extends Rule[LogicalPlan] {
-
- override def apply(plan: LogicalPlan): LogicalPlan =
- plan.transformWithPruning (_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
- case replace @ ReplaceData(_, cond, _, _, _, _, _) =>
- replaceStaticInvoke(replace, cond, newCond => replace.copy(condition = newCond))
-
- case join @ Join(_, _, _, Some(cond), _) =>
- replaceStaticInvoke(join, cond, newCond => join.copy(condition = Some(newCond)))
-
- case filter @ Filter(cond, _) =>
- replaceStaticInvoke(filter, cond, newCond => filter.copy(condition = newCond))
- }
-
- private def replaceStaticInvoke[T <: LogicalPlan](
- node: T,
- condition: Expression,
- copy: Expression => T): T = {
- val newCondition = replaceStaticInvoke(condition)
- if (newCondition fastEquals condition) node else copy(newCondition)
- }
-
- private def replaceStaticInvoke(condition: Expression): Expression = {
- condition.transformWithPruning(_.containsAnyPattern(BINARY_COMPARISON, IN, INSET)) {
- case in @ In(value: StaticInvoke, _) if canReplace(value) =>
- in.copy(value = replaceStaticInvoke(value))
-
- case in @ InSet(value: StaticInvoke, _) if canReplace(value) =>
- in.copy(child = replaceStaticInvoke(value))
-
- case c @ BinaryComparison(left: StaticInvoke, right) if canReplace(left) && right.foldable =>
- c.withNewChildren(Seq(replaceStaticInvoke(left), right))
-
- case c @ BinaryComparison(left, right: StaticInvoke) if canReplace(right) && left.foldable =>
- c.withNewChildren(Seq(left, replaceStaticInvoke(right)))
- }
- }
-
- private def replaceStaticInvoke(invoke: StaticInvoke): Expression = {
- // Adaptive from `resolveV2Function` in org.apache.spark.sql.catalyst.analysis.ResolveFunctions
- val unbound = SparkFunctions.loadFunctionByClass(invoke.staticObject)
- if (unbound == null) {
- return invoke
- }
-
- val inputType = StructType(invoke.arguments.zipWithIndex.map {
- case (exp, pos) => StructField(s"_$pos", exp.dataType, exp.nullable)
- })
-
- val bound = try {
- unbound.bind(inputType)
- } catch {
- case _: Exception =>
- return invoke
- }
-
- if (bound.inputTypes().length != invoke.arguments.length) {
- return invoke
- }
-
- bound match {
- case scalarFunc: ScalarFunction[_] =>
- ApplyFunctionExpression(scalarFunc, invoke.arguments)
- case _ => invoke
- }
- }
-
- @inline
- private def canReplace(invoke: StaticInvoke): Boolean = {
- invoke.functionName == ScalarFunction.MAGIC_METHOD_NAME && !invoke.foldable
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
deleted file mode 100644
index d450d043bc..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.parser.extensions
-
-import java.util.Locale
-import org.antlr.v4.runtime._
-import org.antlr.v4.runtime.atn.PredictionMode
-import org.antlr.v4.runtime.misc.Interval
-import org.antlr.v4.runtime.misc.ParseCancellationException
-import org.antlr.v4.runtime.tree.TerminalNodeImpl
-import org.apache.iceberg.common.DynConstructors
-import org.apache.iceberg.spark.ExtendedParser
-import org.apache.iceberg.spark.ExtendedParser.RawOrderField
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.procedures.SparkProcedures
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
-import org.apache.spark.sql.catalyst.analysis.RewriteViewCommands
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.trees.Origin
-import org.apache.spark.sql.connector.catalog.Table
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.VariableSubstitution
-import org.apache.spark.sql.types.DataType
-import org.apache.spark.sql.types.StructType
-import scala.jdk.CollectionConverters._
-import scala.util.Try
-
-class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserInterface with ExtendedParser {
-
- import IcebergSparkSqlExtensionsParser._
-
- private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get)
- private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate)
-
- /**
- * Parse a string to a DataType.
- */
- override def parseDataType(sqlText: String): DataType = {
- delegate.parseDataType(sqlText)
- }
-
- /**
- * Parse a string to a raw DataType without CHAR/VARCHAR replacement.
- */
- def parseRawDataType(sqlText: String): DataType = throw new UnsupportedOperationException()
-
- /**
- * Parse a string to an Expression.
- */
- override def parseExpression(sqlText: String): Expression = {
- delegate.parseExpression(sqlText)
- }
-
- /**
- * Parse a string to a TableIdentifier.
- */
- override def parseTableIdentifier(sqlText: String): TableIdentifier = {
- delegate.parseTableIdentifier(sqlText)
- }
-
- /**
- * Parse a string to a FunctionIdentifier.
- */
- override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = {
- delegate.parseFunctionIdentifier(sqlText)
- }
-
- /**
- * Parse a string to a multi-part identifier.
- */
- override def parseMultipartIdentifier(sqlText: String): Seq[String] = {
- delegate.parseMultipartIdentifier(sqlText)
- }
-
- /**
- * Creates StructType for a given SQL string, which is a comma separated list of field
- * definitions which will preserve the correct Hive metadata.
- */
- override def parseTableSchema(sqlText: String): StructType = {
- delegate.parseTableSchema(sqlText)
- }
-
- override def parseSortOrder(sqlText: String): java.util.List[RawOrderField] = {
- val fields = parse(sqlText) { parser => astBuilder.visitSingleOrder(parser.singleOrder()) }
- fields.map { field =>
- val (term, direction, order) = field
- new RawOrderField(term, direction, order)
- }.asJava
- }
-
- /**
- * Parse a string to a LogicalPlan.
- */
- override def parsePlan(sqlText: String): LogicalPlan = {
- val sqlTextAfterSubstitution = substitutor.substitute(sqlText)
- if (isIcebergCommand(sqlTextAfterSubstitution)) {
- parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }.asInstanceOf[LogicalPlan]
- } else {
- RewriteViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText))
- }
- }
-
- private def isIcebergCommand(sqlText: String): Boolean = {
- val normalized = sqlText.toLowerCase(Locale.ROOT).trim()
- // Strip simple SQL comments that terminate a line, e.g. comments starting with `--` .
- .replaceAll("--.*?\\n", " ")
- // Strip newlines.
- .replaceAll("\\s+", " ")
- // Strip comments of the form /* ... */. This must come after stripping newlines so that
- // comments that span multiple lines are caught.
- .replaceAll("/\\*.*?\\*/", " ")
- // Strip backtick then `system`.`ancestors_of` changes to system.ancestors_of
- .replaceAll("`", "")
- .trim()
-
- isIcebergProcedure(normalized) || (
- normalized.startsWith("alter table") && (
- normalized.contains("add partition field") ||
- normalized.contains("drop partition field") ||
- normalized.contains("replace partition field") ||
- normalized.contains("write ordered by") ||
- normalized.contains("write locally ordered by") ||
- normalized.contains("write distributed by") ||
- normalized.contains("write unordered") ||
- normalized.contains("set identifier fields") ||
- normalized.contains("drop identifier fields") ||
- isSnapshotRefDdl(normalized)))
- }
-
- // All builtin Iceberg procedures are under the 'system' namespace
- private def isIcebergProcedure(normalized: String): Boolean = {
- normalized.startsWith("call") &&
- SparkProcedures.names().asScala.map("system." + _).exists(normalized.contains)
- }
-
- private def isSnapshotRefDdl(normalized: String): Boolean = {
- normalized.contains("create branch") ||
- normalized.contains("replace branch") ||
- normalized.contains("create tag") ||
- normalized.contains("replace tag") ||
- normalized.contains("drop branch") ||
- normalized.contains("drop tag")
- }
-
- protected def parse[T](command: String)(toResult: IcebergSqlExtensionsParser => T): T = {
- val lexer = new IcebergSqlExtensionsLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
- lexer.removeErrorListeners()
- lexer.addErrorListener(IcebergParseErrorListener)
-
- val tokenStream = new CommonTokenStream(lexer)
- val parser = new IcebergSqlExtensionsParser(tokenStream)
- parser.addParseListener(IcebergSqlExtensionsPostProcessor)
- parser.removeErrorListeners()
- parser.addErrorListener(IcebergParseErrorListener)
-
- // https://github.com/antlr/antlr4/issues/192#issuecomment-15238595
- // Save a great deal of time on correct inputs by using a two-stage parsing strategy.
- try {
- try {
- // first, try parsing with potentially faster SLL mode and BailErrorStrategy
- parser.setErrorHandler(new BailErrorStrategy)
- parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
- toResult(parser)
- }
- catch {
- case _: ParseCancellationException =>
- // if we fail, parse with LL mode with DefaultErrorStrategy
- tokenStream.seek(0) // rewind input stream
- parser.reset()
-
- // Try Again.
- parser.setErrorHandler(new DefaultErrorStrategy)
- parser.getInterpreter.setPredictionMode(PredictionMode.LL)
- toResult(parser)
- }
- }
- catch {
- case e: IcebergParseException if e.command.isDefined =>
- throw e
- case e: IcebergParseException =>
- throw e.withCommand(command)
- case e: AnalysisException =>
- val position = Origin(e.line, e.startPosition)
- throw new IcebergParseException(Option(command), e.message, position, position)
- }
- }
-
- override def parseQuery(sqlText: String): LogicalPlan = {
- parsePlan(sqlText)
- }
-}
-
-object IcebergSparkSqlExtensionsParser {
- private val substitutorCtor: DynConstructors.Ctor[VariableSubstitution] =
- DynConstructors.builder()
- .impl(classOf[VariableSubstitution])
- .impl(classOf[VariableSubstitution], classOf[SQLConf])
- .build()
-}
-
-/* Copied from Apache Spark's to avoid dependency on Spark Internals */
-class UpperCaseCharStream(wrapped: CodePointCharStream) extends CharStream {
- override def consume(): Unit = wrapped.consume
- override def getSourceName(): String = wrapped.getSourceName
- override def index(): Int = wrapped.index
- override def mark(): Int = wrapped.mark
- override def release(marker: Int): Unit = wrapped.release(marker)
- override def seek(where: Int): Unit = wrapped.seek(where)
- override def size(): Int = wrapped.size
-
- override def getText(interval: Interval): String = wrapped.getText(interval)
-
- // scalastyle:off
- override def LA(i: Int): Int = {
- val la = wrapped.LA(i)
- if (la == 0 || la == IntStream.EOF) la
- else Character.toUpperCase(la)
- }
- // scalastyle:on
-}
-
-/**
- * The post-processor validates & cleans-up the parse tree during the parse process.
- */
-case object IcebergSqlExtensionsPostProcessor extends IcebergSqlExtensionsBaseListener {
-
- /** Remove the back ticks from an Identifier. */
- override def exitQuotedIdentifier(ctx: QuotedIdentifierContext): Unit = {
- replaceTokenByIdentifier(ctx, 1) { token =>
- // Remove the double back ticks in the string.
- token.setText(token.getText.replace("``", "`"))
- token
- }
- }
-
- /** Treat non-reserved keywords as Identifiers. */
- override def exitNonReserved(ctx: NonReservedContext): Unit = {
- replaceTokenByIdentifier(ctx, 0)(identity)
- }
-
- private def replaceTokenByIdentifier(
- ctx: ParserRuleContext,
- stripMargins: Int)(
- f: CommonToken => CommonToken = identity): Unit = {
- val parent = ctx.getParent
- parent.removeLastChild()
- val token = ctx.getChild(0).getPayload.asInstanceOf[Token]
- val newToken = new CommonToken(
- new org.antlr.v4.runtime.misc.Pair(token.getTokenSource, token.getInputStream),
- IcebergSqlExtensionsParser.IDENTIFIER,
- token.getChannel,
- token.getStartIndex + stripMargins,
- token.getStopIndex - stripMargins)
- parent.addChild(new TerminalNodeImpl(f(newToken)))
- }
-}
-
-/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
-case object IcebergParseErrorListener extends BaseErrorListener {
- override def syntaxError(
- recognizer: Recognizer[_, _],
- offendingSymbol: scala.Any,
- line: Int,
- charPositionInLine: Int,
- msg: String,
- e: RecognitionException): Unit = {
- val (start, stop) = offendingSymbol match {
- case token: CommonToken =>
- val start = Origin(Some(line), Some(token.getCharPositionInLine))
- val length = token.getStopIndex - token.getStartIndex + 1
- val stop = Origin(Some(line), Some(token.getCharPositionInLine + length))
- (start, stop)
- case _ =>
- val start = Origin(Some(line), Some(charPositionInLine))
- (start, start)
- }
- throw new IcebergParseException(None, msg, start, stop)
- }
-}
-
-/**
- * Copied from Apache Spark
- * A [[ParseException]] is an [[AnalysisException]] that is thrown during the parse process. It
- * contains fields and an extended error message that make reporting and diagnosing errors easier.
- */
-class IcebergParseException(
- val command: Option[String],
- message: String,
- val start: Origin,
- val stop: Origin) extends AnalysisException(message, start.line, start.startPosition) {
-
- def this(message: String, ctx: ParserRuleContext) = {
- this(Option(IcebergParserUtils.command(ctx)),
- message,
- IcebergParserUtils.position(ctx.getStart),
- IcebergParserUtils.position(ctx.getStop))
- }
-
- override def getMessage: String = {
- val builder = new StringBuilder
- builder ++= "\n" ++= message
- start match {
- case Origin(Some(l), Some(p), Some(startIndex), Some(stopIndex), Some(sqlText), Some(objectType),
- Some(objectName), _, _) =>
- builder ++= s"(line $l, pos $p)\n"
- command.foreach { cmd =>
- val (above, below) = cmd.split("\n").splitAt(l)
- builder ++= "\n== SQL ==\n"
- above.foreach(builder ++= _ += '\n')
- builder ++= (0 until p).map(_ => "-").mkString("") ++= "^^^\n"
- below.foreach(builder ++= _ += '\n')
- }
- case _ =>
- command.foreach { cmd =>
- builder ++= "\n== SQL ==\n" ++= cmd
- }
- }
- builder.toString
- }
-
- def withCommand(cmd: String): IcebergParseException = {
- new IcebergParseException(Option(cmd), message, start, stop)
- }
-}
\ No newline at end of file
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
deleted file mode 100644
index b95fc7755f..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
+++ /dev/null
@@ -1,387 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.parser.extensions
-
-import java.util.Locale
-import java.util.concurrent.TimeUnit
-import org.antlr.v4.runtime._
-import org.antlr.v4.runtime.misc.Interval
-import org.antlr.v4.runtime.tree.ParseTree
-import org.antlr.v4.runtime.tree.TerminalNode
-import org.apache.iceberg.DistributionMode
-import org.apache.iceberg.NullOrder
-import org.apache.iceberg.SortDirection
-import org.apache.iceberg.expressions.Term
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.spark.sql.catalyst.analysis.IcebergAnalysisException
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergParserUtils.withOrigin
-import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser._
-import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.BranchOptions
-import org.apache.spark.sql.catalyst.plans.logical.CallArgument
-import org.apache.spark.sql.catalyst.plans.logical.CallStatement
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag
-import org.apache.spark.sql.catalyst.plans.logical.DropBranch
-import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.DropTag
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.NamedArgument
-import org.apache.spark.sql.catalyst.plans.logical.PositionalArgument
-import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
-import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
-import org.apache.spark.sql.catalyst.plans.logical.TagOptions
-import org.apache.spark.sql.catalyst.trees.CurrentOrigin
-import org.apache.spark.sql.catalyst.trees.Origin
-import org.apache.spark.sql.connector.expressions
-import org.apache.spark.sql.connector.expressions.ApplyTransform
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.IdentityTransform
-import org.apache.spark.sql.connector.expressions.LiteralValue
-import org.apache.spark.sql.connector.expressions.Transform
-import scala.jdk.CollectionConverters._
-
-class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
-
- private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] = list.asScala
- private def toSeq[T](list: java.util.List[T]): Seq[T] = toBuffer(list).toSeq
-
- /**
- * Create a [[CallStatement]] for a stored procedure call.
- */
- override def visitCall(ctx: CallContext): CallStatement = withOrigin(ctx) {
- val name = toSeq(ctx.multipartIdentifier.parts).map(_.getText)
- val args = toSeq(ctx.callArgument).map(typedVisit[CallArgument])
- CallStatement(name, args)
- }
-
- /**
- * Create an ADD PARTITION FIELD logical command.
- */
- override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField = withOrigin(ctx) {
- AddPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform),
- Option(ctx.name).map(_.getText))
- }
-
- /**
- * Create a DROP PARTITION FIELD logical command.
- */
- override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField = withOrigin(ctx) {
- DropPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform))
- }
-
- /**
- * Create a CREATE OR REPLACE BRANCH logical command.
- */
- override def visitCreateOrReplaceBranch(ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
- val createOrReplaceBranchClause = ctx.createReplaceBranchClause()
-
- val branchName = createOrReplaceBranchClause.identifier()
- val branchOptionsContext = Option(createOrReplaceBranchClause.branchOptions())
- val snapshotId = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotId()))
- .map(_.getText.toLong)
- val snapshotRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
- val minSnapshotsToKeep = snapshotRetention.flatMap(retention => Option(retention.minSnapshotsToKeep()))
- .map(minSnapshots => minSnapshots.number().getText.toLong)
- val maxSnapshotAgeMs = snapshotRetention
- .flatMap(retention => Option(retention.maxSnapshotAge()))
- .map(retention => TimeUnit.valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
- .toMillis(retention.number().getText.toLong))
- val branchRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
- val branchRefAgeMs = branchRetention.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
- val create = createOrReplaceBranchClause.CREATE() != null
- val replace = ctx.createReplaceBranchClause().REPLACE() != null
- val ifNotExists = createOrReplaceBranchClause.EXISTS() != null
-
- val branchOptions = BranchOptions(
- snapshotId,
- minSnapshotsToKeep,
- maxSnapshotAgeMs,
- branchRefAgeMs
- )
-
- CreateOrReplaceBranch(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- branchName.getText,
- branchOptions,
- create,
- replace,
- ifNotExists)
- }
-
- /**
- * Create an CREATE OR REPLACE TAG logical command.
- */
- override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag = withOrigin(ctx) {
- val createTagClause = ctx.createReplaceTagClause()
-
- val tagName = createTagClause.identifier().getText
-
- val tagOptionsContext = Option(createTagClause.tagOptions())
- val snapshotId = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.snapshotId()))
- .map(_.getText.toLong)
- val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
- val tagRefAgeMs = tagRetain.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
- val tagOptions = TagOptions(
- snapshotId,
- tagRefAgeMs
- )
-
- val create = createTagClause.CREATE() != null
- val replace = createTagClause.REPLACE() != null
- val ifNotExists = createTagClause.EXISTS() != null
-
- CreateOrReplaceTag(typedVisit[Seq[String]](ctx.multipartIdentifier),
- tagName,
- tagOptions,
- create,
- replace,
- ifNotExists)
- }
-
- /**
- * Create an DROP BRANCH logical command.
- */
- override def visitDropBranch(ctx: DropBranchContext): DropBranch = withOrigin(ctx) {
- DropBranch(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
- }
-
- /**
- * Create an DROP TAG logical command.
- */
- override def visitDropTag(ctx: DropTagContext): DropTag = withOrigin(ctx) {
- DropTag(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
- }
-
- /**
- * Create an REPLACE PARTITION FIELD logical command.
- */
- override def visitReplacePartitionField(ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
- ReplacePartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform(0)),
- typedVisit[Transform](ctx.transform(1)),
- Option(ctx.name).map(_.getText))
- }
-
- /**
- * Create an SET IDENTIFIER FIELDS logical command.
- */
- override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields = withOrigin(ctx) {
- SetIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
-
- /**
- * Create an DROP IDENTIFIER FIELDS logical command.
- */
- override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields = withOrigin(ctx) {
- DropIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
-
- /**
- * Create a [[SetWriteDistributionAndOrdering]] for changing the write distribution and ordering.
- */
- override def visitSetWriteDistributionAndOrdering(
- ctx: SetWriteDistributionAndOrderingContext): SetWriteDistributionAndOrdering = {
-
- val tableName = typedVisit[Seq[String]](ctx.multipartIdentifier)
-
- val (distributionSpec, orderingSpec) = toDistributionAndOrderingSpec(ctx.writeSpec)
-
- if (distributionSpec == null && orderingSpec == null) {
- throw new IcebergAnalysisException(
- "ALTER TABLE has no changes: missing both distribution and ordering clauses")
- }
-
- val distributionMode = if (distributionSpec != null) {
- Some(DistributionMode.HASH)
- } else if (orderingSpec.UNORDERED != null) {
- Some(DistributionMode.NONE)
- } else if (orderingSpec.LOCALLY() != null) {
- None
- } else {
- Some(DistributionMode.RANGE)
- }
-
- val ordering = if (orderingSpec != null && orderingSpec.order != null) {
- toSeq(orderingSpec.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
- } else {
- Seq.empty
- }
-
- SetWriteDistributionAndOrdering(tableName, distributionMode, ordering)
- }
-
- private def toDistributionAndOrderingSpec(
- writeSpec: WriteSpecContext): (WriteDistributionSpecContext, WriteOrderingSpecContext) = {
-
- if (writeSpec.writeDistributionSpec.size > 1) {
- throw new IcebergAnalysisException("ALTER TABLE contains multiple distribution clauses")
- }
-
- if (writeSpec.writeOrderingSpec.size > 1) {
- throw new IcebergAnalysisException("ALTER TABLE contains multiple ordering clauses")
- }
-
- val distributionSpec = toBuffer(writeSpec.writeDistributionSpec).headOption.orNull
- val orderingSpec = toBuffer(writeSpec.writeOrderingSpec).headOption.orNull
-
- (distributionSpec, orderingSpec)
- }
-
- /**
- * Create an order field.
- */
- override def visitOrderField(ctx: OrderFieldContext): (Term, SortDirection, NullOrder) = {
- val term = Spark3Util.toIcebergTerm(typedVisit[Transform](ctx.transform))
- val direction = Option(ctx.ASC).map(_ => SortDirection.ASC)
- .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
- .getOrElse(SortDirection.ASC)
- val nullOrder = Option(ctx.FIRST).map(_ => NullOrder.NULLS_FIRST)
- .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
- .getOrElse(if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
- (term, direction, nullOrder)
- }
-
- /**
- * Create an IdentityTransform for a column reference.
- */
- override def visitIdentityTransform(ctx: IdentityTransformContext): Transform = withOrigin(ctx) {
- IdentityTransform(FieldReference(typedVisit[Seq[String]](ctx.multipartIdentifier())))
- }
-
- /**
- * Create a named Transform from argument expressions.
- */
- override def visitApplyTransform(ctx: ApplyTransformContext): Transform = withOrigin(ctx) {
- val args = toSeq(ctx.arguments).map(typedVisit[expressions.Expression])
- ApplyTransform(ctx.transformName.getText, args)
- }
-
- /**
- * Create a transform argument from a column reference or a constant.
- */
- override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression = withOrigin(ctx) {
- val reference = Option(ctx.multipartIdentifier())
- .map(typedVisit[Seq[String]])
- .map(FieldReference(_))
- val literal = Option(ctx.constant)
- .map(visitConstant)
- .map(lit => LiteralValue(lit.value, lit.dataType))
- reference.orElse(literal)
- .getOrElse(throw new IcebergParseException(s"Invalid transform argument", ctx))
- }
-
- /**
- * Return a multi-part identifier as Seq[String].
- */
- override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) {
- toSeq(ctx.parts).map(_.getText)
- }
-
- override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] = withOrigin(ctx) {
- toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
- }
-
- /**
- * Create a positional argument in a stored procedure call.
- */
- override def visitPositionalArgument(ctx: PositionalArgumentContext): CallArgument = withOrigin(ctx) {
- val expr = typedVisit[Expression](ctx.expression)
- PositionalArgument(expr)
- }
-
- /**
- * Create a named argument in a stored procedure call.
- */
- override def visitNamedArgument(ctx: NamedArgumentContext): CallArgument = withOrigin(ctx) {
- val name = ctx.identifier.getText
- val expr = typedVisit[Expression](ctx.expression)
- NamedArgument(name, expr)
- }
-
- override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) {
- visit(ctx.statement).asInstanceOf[LogicalPlan]
- }
-
- def visitConstant(ctx: ConstantContext): Literal = {
- delegate.parseExpression(ctx.getText).asInstanceOf[Literal]
- }
-
- override def visitExpression(ctx: ExpressionContext): Expression = {
- // reconstruct the SQL string and parse it using the main Spark parser
- // while we can avoid the logic to build Spark expressions, we still have to parse them
- // we cannot call ctx.getText directly since it will not render spaces correctly
- // that's why we need to recurse down the tree in reconstructSqlString
- val sqlString = reconstructSqlString(ctx)
- delegate.parseExpression(sqlString)
- }
-
- private def reconstructSqlString(ctx: ParserRuleContext): String = {
- toBuffer(ctx.children).map {
- case c: ParserRuleContext => reconstructSqlString(c)
- case t: TerminalNode => t.getText
- }.mkString(" ")
- }
-
- private def typedVisit[T](ctx: ParseTree): T = {
- ctx.accept(this).asInstanceOf[T]
- }
-}
-
-/* Partially copied from Apache Spark's Parser to avoid dependency on Spark Internals */
-object IcebergParserUtils {
-
- private[sql] def withOrigin[T](ctx: ParserRuleContext)(f: => T): T = {
- val current = CurrentOrigin.get
- CurrentOrigin.set(position(ctx.getStart))
- try {
- f
- } finally {
- CurrentOrigin.set(current)
- }
- }
-
- private[sql] def position(token: Token): Origin = {
- val opt = Option(token)
- Origin(opt.map(_.getLine), opt.map(_.getCharPositionInLine))
- }
-
- /** Get the command which created the token. */
- private[sql] def command(ctx: ParserRuleContext): String = {
- val stream = ctx.getStart.getInputStream
- stream.getText(Interval.of(0, stream.size() - 1))
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
deleted file mode 100644
index e8b1b29411..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"AddPartitionField ${table.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
deleted file mode 100644
index 4d7e0a086b..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-case class BranchOptions (snapshotId: Option[Long], numSnapshots: Option[Long],
- snapshotRetain: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
deleted file mode 100644
index b7981a3c7a..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class CreateOrReplaceBranch(
- table: Seq[String],
- branch: String,
- branchOptions: BranchOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"CreateOrReplaceBranch branch: ${branch} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
deleted file mode 100644
index 6e7db84a90..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class CreateOrReplaceTag(
- table: Seq[String],
- tag: String,
- tagOptions: TagOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"CreateOrReplaceTag tag: ${tag} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
deleted file mode 100644
index bee0b0fae6..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class DropBranch(table: Seq[String], branch: String, ifExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropBranch branch: ${branch} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
deleted file mode 100644
index 29dd686a0f..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class DropIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropIdentifierFields ${table.quoted} (${fields.quoted})"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
deleted file mode 100644
index fb14513241..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class DropPartitionField(table: Seq[String], transform: Transform) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropPartitionField ${table.quoted} ${transform.describe}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
deleted file mode 100644
index 7e4b38e74d..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-
-case class DropTag(table: Seq[String], tag: String, ifExists: Boolean) extends LeafCommand {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"DropTag tag: ${tag} for table: ${table.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/IcebergCall.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/IcebergCall.scala
deleted file mode 100644
index 032eac1397..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/IcebergCall.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.types.DataTypeUtils
-import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.connector.iceberg.catalog.Procedure
-
-case class IcebergCall(procedure: Procedure, args: Seq[Expression]) extends LeafCommand {
- override lazy val output: Seq[Attribute] = DataTypeUtils.toAttributes(procedure.outputType)
-
- override def simpleString(maxFields: Int): String = {
- s"IcebergCall${truncatedString(output.toSeq, "[", ", ", "]", maxFields)} ${procedure.description}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
deleted file mode 100644
index 8c660c6f37..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class ReplacePartitionField(
- table: Seq[String],
- transformFrom: Transform,
- transformTo: Transform,
- name: Option[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"ReplacePartitionField ${table.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
deleted file mode 100644
index a5fa28a617..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class SetIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override def simpleString(maxFields: Int): String = {
- s"SetIdentifierFields ${table.quoted} (${fields.quoted})"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
deleted file mode 100644
index 85e3b95f4a..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-case class TagOptions(snapshotId: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
deleted file mode 100644
index be15f32bc1..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical
-
-import org.apache.spark.sql.catalyst.expressions.Expression
-
-/**
- * A CALL statement, as parsed from SQL.
- */
-case class CallStatement(name: Seq[String], args: Seq[CallArgument]) extends LeafParsedStatement
-
-/**
- * An argument in a CALL statement.
- */
-sealed trait CallArgument {
- def expr: Expression
-}
-
-/**
- * An argument in a CALL statement identified by name.
- */
-case class NamedArgument(name: String, expr: Expression) extends CallArgument
-
-/**
- * An argument in a CALL statement identified by position.
- */
-case class PositionalArgument(expr: Expression) extends CallArgument
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
deleted file mode 100644
index 9366d5efe1..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical.views
-
-import org.apache.spark.sql.catalyst.plans.logical.BinaryCommand
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-
-case class CreateIcebergView(
- child: LogicalPlan,
- queryText: String,
- query: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String] = Seq.empty,
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean,
- rewritten: Boolean = false) extends BinaryCommand {
- override def left: LogicalPlan = child
-
- override def right: LogicalPlan = query
-
- override protected def withNewChildrenInternal(
- newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
- copy(child = newLeft, query = newRight)
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
deleted file mode 100644
index 275dba6fbf..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical.views
-
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
-
-case class DropIcebergView(
- child: LogicalPlan,
- ifExists: Boolean) extends UnaryCommand {
- override protected def withNewChildInternal(newChild: LogicalPlan): DropIcebergView =
- copy(child = newChild)
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
deleted file mode 100644
index b9c05ff006..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.catalyst.plans.logical.views
-
-import org.apache.spark.sql.catalyst.analysis.LeafNodeWithoutStats
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-case class ResolvedV2View(
- catalog: ViewCatalog,
- identifier: Identifier) extends LeafNodeWithoutStats {
- override def output: Seq[Attribute] = Nil
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
deleted file mode 100644
index b09c27acdc..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.spark.sql.catalyst.plans.logical.views
-
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.ShowViews
-import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
-
-case class ShowIcebergViews(
- namespace: LogicalPlan,
- pattern: Option[String],
- override val output: Seq[Attribute] = ShowViews.getOutputAttrs) extends UnaryCommand {
- override def child: LogicalPlan = namespace
-
- override protected def withNewChildInternal(newChild: LogicalPlan): ShowIcebergViews =
- copy(namespace = newChild)
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
deleted file mode 100644
index 55f327f7e4..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class AddPartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transform: Transform,
- name: Option[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- iceberg.table.updateSpec()
- .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
- .commit()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot add partition field to non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"AddPartitionField ${catalog.name}.${ident.quoted} ${name.map(n => s"$n=").getOrElse("")}${transform.describe}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
deleted file mode 100644
index b103d1ee2c..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.connector.catalog.ViewChange
-
-
-case class AlterV2ViewSetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- properties: Map[String, String]) extends LeafV2CommandExec {
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- val changes = properties.map {
- case (property, value) => ViewChange.setProperty(property, value)
- }.toSeq
-
- catalog.alterView(ident, changes: _*)
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"AlterV2ViewSetProperties: ${ident}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
deleted file mode 100644
index a4103fede2..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.IcebergAnalysisException
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.connector.catalog.ViewChange
-
-
-case class AlterV2ViewUnsetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- propertyKeys: Seq[String],
- ifExists: Boolean) extends LeafV2CommandExec {
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- if (!ifExists) {
- propertyKeys.filterNot(catalog.loadView(ident).properties.containsKey).foreach { property =>
- throw new IcebergAnalysisException(s"Cannot remove property that is not set: '$property'")
- }
- }
-
- val changes = propertyKeys.map(ViewChange.removeProperty)
- catalog.alterView(ident, changes: _*)
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"AlterV2ViewUnsetProperties: ${ident}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
deleted file mode 100644
index f66962a8c4..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.connector.iceberg.catalog.Procedure
-import scala.collection.compat.immutable.ArraySeq
-
-case class CallExec(
- output: Seq[Attribute],
- procedure: Procedure,
- input: InternalRow) extends LeafV2CommandExec {
-
- override protected def run(): Seq[InternalRow] = {
- ArraySeq.unsafeWrapArray(procedure.call(input))
- }
-
- override def simpleString(maxFields: Int): String = {
- s"CallExec${truncatedString(output, "[", ", ", "]", maxFields)} ${procedure.description}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
deleted file mode 100644
index 2be406e7f3..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.BranchOptions
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class CreateOrReplaceBranchExec(
- catalog: TableCatalog,
- ident: Identifier,
- branch: String,
- branchOptions: BranchOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val snapshotId: java.lang.Long = branchOptions.snapshotId
- .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId()))
- .map(java.lang.Long.valueOf)
- .orNull
-
- val manageSnapshots = iceberg.table().manageSnapshots()
- val refExists = null != iceberg.table().refs().get(branch)
-
- def safeCreateBranch(): Unit = {
- if (snapshotId == null) {
- manageSnapshots.createBranch(branch)
- } else {
- manageSnapshots.createBranch(branch, snapshotId)
- }
- }
-
- if (create && replace && !refExists) {
- safeCreateBranch()
- } else if (replace) {
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete replace branch operation on %s, main has no snapshot", ident)
- manageSnapshots.replaceBranch(branch, snapshotId)
- } else {
- if (refExists && ifNotExists) {
- return Nil
- }
-
- safeCreateBranch()
- }
-
- if (branchOptions.numSnapshots.nonEmpty) {
- manageSnapshots.setMinSnapshotsToKeep(branch, branchOptions.numSnapshots.get.toInt)
- }
-
- if (branchOptions.snapshotRetain.nonEmpty) {
- manageSnapshots.setMaxSnapshotAgeMs(branch, branchOptions.snapshotRetain.get)
- }
-
- if (branchOptions.snapshotRefRetain.nonEmpty) {
- manageSnapshots.setMaxRefAgeMs(branch, branchOptions.snapshotRefRetain.get)
- }
-
- manageSnapshots.commit()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot create or replace branch on non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"CreateOrReplace branch: $branch for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
deleted file mode 100644
index 372cd75486..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.TagOptions
-import org.apache.spark.sql.connector.catalog._
-
-case class CreateOrReplaceTagExec(
- catalog: TableCatalog,
- ident: Identifier,
- tag: String,
- tagOptions: TagOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val snapshotId: java.lang.Long = tagOptions.snapshotId
- .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId()))
- .map(java.lang.Long.valueOf)
- .orNull
-
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete create or replace tag operation on %s, main has no snapshot", ident)
-
- val manageSnapshot = iceberg.table.manageSnapshots()
- val refExists = null != iceberg.table().refs().get(tag)
-
- if (create && replace && !refExists) {
- manageSnapshot.createTag(tag, snapshotId)
- } else if (replace) {
- manageSnapshot.replaceTag(tag, snapshotId)
- } else {
- if (refExists && ifNotExists) {
- return Nil
- }
-
- manageSnapshot.createTag(tag, snapshotId)
- }
-
- if (tagOptions.snapshotRefRetain.nonEmpty) {
- manageSnapshot.setMaxRefAgeMs(tag, tagOptions.snapshotRefRetain.get)
- }
-
- manageSnapshot.commit()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot create tag to non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"Create tag: $tag for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
deleted file mode 100644
index 9015fb338e..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.SupportsReplaceView
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.NoSuchViewException
-import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.connector.catalog.ViewInfo
-import org.apache.spark.sql.types.StructType
-import scala.collection.JavaConverters._
-
-
-case class CreateV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- queryText: String,
- viewSchema: StructType,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String],
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean) extends LeafV2CommandExec {
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name
- val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
- val currentNamespace = session.sessionState.catalogManager.currentNamespace
-
- val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION
- val newProperties = properties ++
- comment.map(ViewCatalog.PROP_COMMENT -> _) +
- (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
- ViewCatalog.PROP_ENGINE_VERSION -> engineVersion)
-
- if (replace) {
- // CREATE OR REPLACE VIEW
- catalog match {
- case c: SupportsReplaceView =>
- try {
- replaceView(c, currentCatalog, currentNamespace, newProperties)
- } catch {
- // view might have been concurrently dropped during replace
- case _: NoSuchViewException =>
- replaceView(c, currentCatalog, currentNamespace, newProperties)
- }
- case _ =>
- if (catalog.viewExists(ident)) {
- catalog.dropView(ident)
- }
-
- createView(currentCatalog, currentNamespace, newProperties)
- }
- } else {
- try {
- // CREATE VIEW [IF NOT EXISTS]
- createView(currentCatalog, currentNamespace, newProperties)
- } catch {
- case _: ViewAlreadyExistsException if allowExisting => // Ignore
- }
- }
-
- Nil
- }
-
- private def replaceView(
- supportsReplaceView: SupportsReplaceView,
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
- supportsReplaceView.replaceView(
- ident,
- queryText,
- currentCatalog,
- currentNamespace,
- viewSchema,
- queryColumnNames.toArray,
- columnAliases.toArray,
- columnComments.map(c => c.orNull).toArray,
- newProperties.asJava)
- }
-
- private def createView(
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
- val viewInfo: ViewInfo = new ViewInfo(
- ident,
- queryText,
- currentCatalog,
- currentNamespace,
- viewSchema,
- queryColumnNames.toArray,
- columnAliases.toArray,
- columnComments.map(c => c.orNull).toArray,
- newProperties.asJava)
- catalog.createView(viewInfo)
- }
-
- override def simpleString(maxFields: Int): String = {
- s"CreateV2ViewExec: ${ident}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
deleted file mode 100644
index 2a5566a15f..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString
-import org.apache.spark.sql.connector.catalog.View
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.execution.LeafExecNode
-import scala.collection.JavaConverters._
-
-case class DescribeV2ViewExec(
- output: Seq[Attribute],
- view: View,
- isExtended: Boolean) extends V2CommandExec with LeafExecNode {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override protected def run(): Seq[InternalRow] = {
- if (isExtended) {
- (describeSchema :+ emptyRow) ++ describeExtended
- } else {
- describeSchema
- }
- }
-
- private def describeSchema: Seq[InternalRow] =
- view.schema().map { column =>
- toCatalystRow(
- column.name,
- column.dataType.simpleString,
- column.getComment().getOrElse(""))
- }
-
- private def emptyRow: InternalRow = toCatalystRow("", "", "")
-
- private def describeExtended: Seq[InternalRow] = {
- val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
- val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
- val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2)
- val viewProperties = properties.toSeq.sortBy(_._1).map {
- case (key, value) =>
- s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
- }.mkString("[", ", ", "]")
-
-
- // omitting view text here because it is shown as
- // part of SHOW CREATE TABLE and can result in weird formatting in the DESCRIBE output
- toCatalystRow("# Detailed View Information", "", "") ::
- toCatalystRow("Comment", view.properties.getOrDefault(ViewCatalog.PROP_COMMENT, ""), "") ::
- toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
- toCatalystRow("View Query Output Columns", outputColumns, "") ::
- toCatalystRow("View Properties", viewProperties, "") ::
- toCatalystRow("Created By", view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""), "") ::
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DescribeV2ViewExec"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
deleted file mode 100644
index ff8f182009..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class DropBranchExec(
- catalog: TableCatalog,
- ident: Identifier,
- branch: String,
- ifExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val ref = iceberg.table().refs().get(branch)
- if (ref != null || !ifExists) {
- iceberg.table().manageSnapshots().removeBranch(branch).commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop branch on non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropBranch branch: ${branch} for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
deleted file mode 100644
index dee778b474..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions
-import org.apache.iceberg.relocated.com.google.common.collect.Sets
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class DropIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val schema = iceberg.table.schema
- val identifierFieldNames = Sets.newHashSet(schema.identifierFieldNames)
-
- for (name <- fields) {
- Preconditions.checkArgument(schema.findField(name) != null,
- "Cannot complete drop identifier fields operation: field %s not found", name)
- Preconditions.checkArgument(identifierFieldNames.contains(name),
- "Cannot complete drop identifier fields operation: %s is not an identifier field", name)
- identifierFieldNames.remove(name)
- }
-
- iceberg.table.updateSchema()
- .setIdentifierFields(identifierFieldNames)
- .commit();
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop identifier fields in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
deleted file mode 100644
index 9a153f0c00..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.IdentityTransform
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class DropPartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transform: Transform) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val schema = iceberg.table.schema
- transform match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
- // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .commit()
-
- case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transform))
- .commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop partition field in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropPartitionField ${catalog.name}.${ident.quoted} ${transform.describe}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
deleted file mode 100644
index 8df88765a9..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class DropTagExec(
- catalog: TableCatalog,
- ident: Identifier,
- tag: String,
- ifExists: Boolean) extends LeafV2CommandExec {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val ref = iceberg.table().refs().get(tag)
- if (ref != null || !ifExists) {
- iceberg.table().manageSnapshots().removeTag(tag).commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot drop tag on non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropTag tag: ${tag} for table: ${ident.quoted}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
deleted file mode 100644
index c35af1486f..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.NoSuchViewException
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-
-case class DropV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- ifExists: Boolean) extends LeafV2CommandExec {
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- val dropped = catalog.dropView(ident)
- if (!dropped && !ifExists) {
- throw new NoSuchViewException(ident)
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"DropV2View: ${ident}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
deleted file mode 100644
index a33dbdb77e..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.SparkCatalog
-import org.apache.iceberg.spark.SparkSessionCatalog
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.IcebergAnalysisException
-import org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier
-import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace
-import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
-import org.apache.spark.sql.catalyst.expressions.PredicateHelper
-import org.apache.spark.sql.catalyst.plans.logical.AddPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceBranch
-import org.apache.spark.sql.catalyst.plans.logical.CreateOrReplaceTag
-import org.apache.spark.sql.catalyst.plans.logical.DescribeRelation
-import org.apache.spark.sql.catalyst.plans.logical.DropBranch
-import org.apache.spark.sql.catalyst.plans.logical.DropIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField
-import org.apache.spark.sql.catalyst.plans.logical.DropTag
-import org.apache.spark.sql.catalyst.plans.logical.IcebergCall
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.plans.logical.OrderAwareCoalesce
-import org.apache.spark.sql.catalyst.plans.logical.RenameTable
-import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField
-import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields
-import org.apache.spark.sql.catalyst.plans.logical.SetViewProperties
-import org.apache.spark.sql.catalyst.plans.logical.SetWriteDistributionAndOrdering
-import org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable
-import org.apache.spark.sql.catalyst.plans.logical.ShowTableProperties
-import org.apache.spark.sql.catalyst.plans.logical.UnsetViewProperties
-import org.apache.spark.sql.catalyst.plans.logical.views.CreateIcebergView
-import org.apache.spark.sql.catalyst.plans.logical.views.DropIcebergView
-import org.apache.spark.sql.catalyst.plans.logical.views.ResolvedV2View
-import org.apache.spark.sql.catalyst.plans.logical.views.ShowIcebergViews
-import org.apache.spark.sql.classic.Strategy
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.execution.OrderAwareCoalesceExec
-import org.apache.spark.sql.execution.SparkPlan
-import scala.jdk.CollectionConverters._
-
-case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy with PredicateHelper {
-
- override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
- case c @ IcebergCall(procedure, args) =>
- val input = buildInternalRow(args)
- CallExec(c.output, procedure, input) :: Nil
-
- case AddPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform, name) =>
- AddPartitionFieldExec(catalog, ident, transform, name) :: Nil
-
- case CreateOrReplaceBranch(
- IcebergCatalogAndIdentifier(catalog, ident), branch, branchOptions, create, replace, ifNotExists) =>
- CreateOrReplaceBranchExec(catalog, ident, branch, branchOptions, create, replace, ifNotExists) :: Nil
-
- case CreateOrReplaceTag(
- IcebergCatalogAndIdentifier(catalog, ident), tag, tagOptions, create, replace, ifNotExists) =>
- CreateOrReplaceTagExec(catalog, ident, tag, tagOptions, create, replace, ifNotExists) :: Nil
-
- case DropBranch(IcebergCatalogAndIdentifier(catalog, ident), branch, ifExists) =>
- DropBranchExec(catalog, ident, branch, ifExists) :: Nil
-
- case DropTag(IcebergCatalogAndIdentifier(catalog, ident), tag, ifExists) =>
- DropTagExec(catalog, ident, tag, ifExists) :: Nil
-
- case DropPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform) =>
- DropPartitionFieldExec(catalog, ident, transform) :: Nil
-
- case ReplacePartitionField(IcebergCatalogAndIdentifier(catalog, ident), transformFrom, transformTo, name) =>
- ReplacePartitionFieldExec(catalog, ident, transformFrom, transformTo, name) :: Nil
-
- case SetIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
- SetIdentifierFieldsExec(catalog, ident, fields) :: Nil
-
- case DropIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
- DropIdentifierFieldsExec(catalog, ident, fields) :: Nil
-
- case SetWriteDistributionAndOrdering(
- IcebergCatalogAndIdentifier(catalog, ident), distributionMode, ordering) =>
- SetWriteDistributionAndOrderingExec(catalog, ident, distributionMode, ordering) :: Nil
-
- case OrderAwareCoalesce(numPartitions, coalescer, child) =>
- OrderAwareCoalesceExec(numPartitions, coalescer, planLater(child)) :: Nil
-
- case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView@true) =>
- val newIdent = Spark3Util.catalogAndIdentifier(spark, newName.toList.asJava)
- if (oldCatalog.name != newIdent.catalog().name()) {
- throw new IcebergAnalysisException(
- s"Cannot move view between catalogs: from=${oldCatalog.name} and to=${newIdent.catalog().name()}")
- }
- RenameV2ViewExec(oldCatalog, oldIdent, newIdent.identifier()) :: Nil
-
- case DropIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), ifExists) =>
- DropV2ViewExec(viewCatalog, ident, ifExists) :: Nil
-
- case CreateIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), queryText, query,
- columnAliases, columnComments, queryColumnNames, comment, properties, allowExisting, replace, _) =>
- CreateV2ViewExec(
- catalog = viewCatalog,
- ident = ident,
- queryText = queryText,
- columnAliases = columnAliases,
- columnComments = columnComments,
- queryColumnNames = queryColumnNames,
- viewSchema = query.schema,
- comment = comment,
- properties = properties,
- allowExisting = allowExisting,
- replace = replace) :: Nil
-
- case DescribeRelation(ResolvedV2View(catalog, ident), _, isExtended, output) =>
- DescribeV2ViewExec(output, catalog.loadView(ident), isExtended) :: Nil
-
- case ShowTableProperties(ResolvedV2View(catalog, ident), propertyKey, output) =>
- ShowV2ViewPropertiesExec(output, catalog.loadView(ident), propertyKey) :: Nil
-
- case ShowIcebergViews(ResolvedNamespace(catalog: ViewCatalog, namespace, _), pattern, output) =>
- ShowV2ViewsExec(output, catalog, namespace, pattern) :: Nil
-
- case ShowCreateTable(ResolvedV2View(catalog, ident), _, output) =>
- ShowCreateV2ViewExec(output, catalog.loadView(ident)) :: Nil
-
- case SetViewProperties(ResolvedV2View(catalog, ident), properties) =>
- AlterV2ViewSetPropertiesExec(catalog, ident, properties) :: Nil
-
- case UnsetViewProperties(ResolvedV2View(catalog, ident), propertyKeys, ifExists) =>
- AlterV2ViewUnsetPropertiesExec(catalog, ident, propertyKeys, ifExists) :: Nil
-
- case _ => Nil
- }
-
- private def buildInternalRow(exprs: Seq[Expression]): InternalRow = {
- val values = new Array[Any](exprs.size)
- for (index <- exprs.indices) {
- values(index) = exprs(index).eval()
- }
- new GenericInternalRow(values)
- }
-
- private object IcebergCatalogAndIdentifier {
- def unapply(identifier: Seq[String]): Option[(TableCatalog, Identifier)] = {
- val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(spark, identifier.asJava)
- catalogAndIdentifier.catalog match {
- case icebergCatalog: SparkCatalog =>
- Some((icebergCatalog, catalogAndIdentifier.identifier))
- case icebergCatalog: SparkSessionCatalog[_] =>
- Some((icebergCatalog, catalogAndIdentifier.identifier))
- case _ =>
- None
- }
- }
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
deleted file mode 100644
index 61d362044c..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-
-case class RenameV2ViewExec(
- catalog: ViewCatalog,
- oldIdent: Identifier,
- newIdent: Identifier) extends LeafV2CommandExec {
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.renameView(oldIdent, newIdent)
-
- Seq.empty
- }
-
-
- override def simpleString(maxFields: Int): String = {
- s"RenameV2View ${oldIdent} to {newIdent}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
deleted file mode 100644
index fcae0a5def..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.Spark3Util
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import org.apache.spark.sql.connector.expressions.FieldReference
-import org.apache.spark.sql.connector.expressions.IdentityTransform
-import org.apache.spark.sql.connector.expressions.Transform
-
-case class ReplacePartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transformFrom: Transform,
- transformTo: Transform,
- name: Option[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val schema = iceberg.table.schema
- transformFrom match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
- // the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
-
- case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transformFrom))
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
- }
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot replace partition field in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"ReplacePartitionField ${catalog.name}.${ident.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
deleted file mode 100644
index b50550ad38..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-import scala.jdk.CollectionConverters._
-
-case class SetIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- iceberg.table.updateSchema()
- .setIdentifierFields(fields.asJava)
- .commit();
- case table =>
- throw new UnsupportedOperationException(s"Cannot set identifier fields in non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- s"SetIdentifierFields ${catalog.name}.${ident.quoted} (${fields.quoted})";
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
deleted file mode 100644
index c9004ddc5b..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.iceberg.DistributionMode
-import org.apache.iceberg.NullOrder
-import org.apache.iceberg.SortDirection
-import org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE
-import org.apache.iceberg.expressions.Term
-import org.apache.iceberg.spark.SparkUtil
-import org.apache.iceberg.spark.source.SparkTable
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.connector.catalog.TableCatalog
-
-case class SetWriteDistributionAndOrderingExec(
- catalog: TableCatalog,
- ident: Identifier,
- distributionMode: Option[DistributionMode],
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafV2CommandExec {
-
- import CatalogV2Implicits._
-
- override lazy val output: Seq[Attribute] = Nil
-
- override protected def run(): Seq[InternalRow] = {
- catalog.loadTable(ident) match {
- case iceberg: SparkTable =>
- val txn = iceberg.table.newTransaction()
-
- val orderBuilder = txn.replaceSortOrder().caseSensitive(SparkUtil.caseSensitive(session))
- sortOrder.foreach {
- case (term, SortDirection.ASC, nullOrder) =>
- orderBuilder.asc(term, nullOrder)
- case (term, SortDirection.DESC, nullOrder) =>
- orderBuilder.desc(term, nullOrder)
- }
- orderBuilder.commit()
-
- distributionMode.foreach { mode =>
- txn.updateProperties()
- .set(WRITE_DISTRIBUTION_MODE, mode.modeName())
- .commit()
- }
-
- txn.commitTransaction()
-
- case table =>
- throw new UnsupportedOperationException(s"Cannot set write order of non-Iceberg table: $table")
- }
-
- Nil
- }
-
- override def simpleString(maxFields: Int): String = {
- val tableIdent = s"${catalog.name}.${ident.quoted}"
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
- s"SetWriteDistributionAndOrdering $tableIdent $distributionMode $order"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
deleted file mode 100644
index 3be0f15031..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString
-import org.apache.spark.sql.connector.catalog.View
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.execution.LeafExecNode
-import scala.collection.JavaConverters._
-
-case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
- extends V2CommandExec with LeafExecNode {
-
- override protected def run(): Seq[InternalRow] = {
- val builder = new StringBuilder
- builder ++= s"CREATE VIEW ${view.name} "
- showColumns(view, builder)
- showComment(view, builder)
- showProperties(view, builder)
- builder ++= s"AS\n${view.query}\n"
-
- Seq(toCatalystRow(builder.toString))
- }
-
- private def showColumns(view: View, builder: StringBuilder): Unit = {
- val columns = concatByMultiLines(
- view.schema().fields
- .map(x => s"${x.name}${x.getComment().map(c => s" COMMENT '$c'").getOrElse("")}"))
- builder ++= columns
- }
-
- private def showComment(view: View, builder: StringBuilder): Unit = {
- Option(view.properties.get(ViewCatalog.PROP_COMMENT))
- .map("COMMENT '" + escapeSingleQuotedString(_) + "'\n")
- .foreach(builder.append)
- }
-
- private def showProperties(
- view: View,
- builder: StringBuilder): Unit = {
- val showProps = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
- if (showProps.nonEmpty) {
- val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map {
- case (key, value) =>
- s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
- }
-
- builder ++= "TBLPROPERTIES "
- builder ++= concatByMultiLines(props)
- }
- }
-
- private def concatByMultiLines(iter: Iterable[String]): String = {
- iter.mkString("(\n ", ",\n ", ")\n")
- }
-
- override def simpleString(maxFields: Int): String = {
- s"ShowCreateV2ViewExec"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
deleted file mode 100644
index 89fafe99ef..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.View
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.execution.LeafExecNode
-import scala.collection.JavaConverters._
-
-case class ShowV2ViewPropertiesExec(
- output: Seq[Attribute],
- view: View,
- propertyKey: Option[String]) extends V2CommandExec with LeafExecNode {
-
- override protected def run(): Seq[InternalRow] = {
- propertyKey match {
- case Some(p) =>
- val propValue = properties.getOrElse(p,
- s"View ${view.name()} does not have property: $p")
- Seq(toCatalystRow(p, propValue))
- case None =>
- properties.map {
- case (k, v) => toCatalystRow(k, v)
- }.toSeq
- }
- }
-
-
- private def properties = {
- view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
- }
-
- override def simpleString(maxFields: Int): String = {
- s"ShowV2ViewPropertiesExec"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
deleted file mode 100644
index d82d13a3ed..0000000000
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.execution.datasources.v2
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.util.StringUtils
-import org.apache.spark.sql.connector.catalog.ViewCatalog
-import org.apache.spark.sql.execution.LeafExecNode
-import org.apache.spark.sql.internal.SQLConf
-import scala.collection.mutable.ArrayBuffer
-
-case class ShowV2ViewsExec(
- output: Seq[Attribute],
- catalog: ViewCatalog,
- namespace: Seq[String],
- pattern: Option[String]) extends V2CommandExec with LeafExecNode {
-
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-
- override protected def run(): Seq[InternalRow] = {
- val rows = new ArrayBuffer[InternalRow]()
-
- // handle GLOBAL VIEWS
- val globalTemp = SQLConf.get.globalTempDatabase
- if (namespace.nonEmpty && globalTemp == namespace.head) {
- pattern.map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
- .getOrElse(session.sessionState.catalog.globalTempViewManager.listViewNames("*"))
- .map(name => rows += toCatalystRow(globalTemp, name, true))
- } else {
- val views = catalog.listViews(namespace: _*)
- views.map { view =>
- if (pattern.map(StringUtils.filterPattern(Seq(view.name()), _).nonEmpty).getOrElse(true)) {
- rows += toCatalystRow(view.namespace().quoted, view.name(), false)
- }
- }
- }
-
- // include TEMP VIEWS
- pattern.map(p => session.sessionState.catalog.listLocalTempViews(p))
- .getOrElse(session.sessionState.catalog.listLocalTempViews("*"))
- .map(v => rows += toCatalystRow(v.database.toArray.quoted, v.table, true))
-
- rows.toSeq
- }
-
- override def simpleString(maxFields: Int): String = {
- s"ShowV2ViewsExec"
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
deleted file mode 100644
index 8918dfec65..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import java.util.Objects;
-
-public class Employee {
- private Integer id;
- private String dep;
-
- public Employee() {}
-
- public Employee(Integer id, String dep) {
- this.id = id;
- this.dep = dep;
- }
-
- public Integer getId() {
- return id;
- }
-
- public void setId(Integer id) {
- this.id = id;
- }
-
- public String getDep() {
- return dep;
- }
-
- public void setDep(String dep) {
- this.dep = dep;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- } else if (other == null || getClass() != other.getClass()) {
- return false;
- }
-
- Employee employee = (Employee) other;
- return Objects.equals(id, employee.id) && Objects.equals(dep, employee.dep);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id, dep);
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
deleted file mode 100644
index 2dd8469300..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS;
-
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.iceberg.CatalogUtil;
-import org.apache.iceberg.hive.HiveCatalog;
-import org.apache.iceberg.hive.TestHiveMetastore;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.spark.CatalogTestBase;
-import org.apache.iceberg.spark.TestBase;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.internal.SQLConf;
-import org.junit.jupiter.api.BeforeAll;
-
-public abstract class ExtensionsTestBase extends CatalogTestBase {
-
- private static final Random RANDOM = ThreadLocalRandom.current();
-
- @BeforeAll
- public static void startMetastoreAndSpark() {
- TestBase.metastore = new TestHiveMetastore();
- metastore.start();
- TestBase.hiveConf = metastore.hiveConf();
-
- TestBase.spark.stop();
-
- TestBase.spark =
- SparkSession.builder()
- .master("local[2]")
- .config("spark.testing", "true")
- .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic")
- .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
- .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname))
- .config("spark.sql.shuffle.partitions", "4")
- .config("spark.sql.hive.metastorePartitionPruningFallbackOnException", "true")
- .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true")
- .config(
- SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean()))
- .enableHiveSupport()
- .getOrCreate();
-
- TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext());
-
- TestBase.catalog =
- (HiveCatalog)
- CatalogUtil.loadCatalog(
- HiveCatalog.class.getName(), "hive", ImmutableMap.of(), hiveConf);
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java
deleted file mode 100644
index de4acd74a7..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ProcedureUtil.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.UUID;
-import org.apache.iceberg.ImmutableGenericPartitionStatisticsFile;
-import org.apache.iceberg.PartitionStatisticsFile;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.PositionOutputStream;
-
-public class ProcedureUtil {
-
- private ProcedureUtil() {}
-
- static PartitionStatisticsFile writePartitionStatsFile(
- long snapshotId, String statsLocation, FileIO fileIO) {
- PositionOutputStream positionOutputStream;
- try {
- positionOutputStream = fileIO.newOutputFile(statsLocation).create();
- positionOutputStream.close();
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
-
- return ImmutableGenericPartitionStatisticsFile.builder()
- .snapshotId(snapshotId)
- .fileSizeInBytes(42L)
- .path(statsLocation)
- .build();
- }
-
- static String statsFileLocation(String tableLocation) {
- String statsFileName = "stats-file-" + UUID.randomUUID();
- return tableLocation.replaceFirst("file:", "") + "/metadata/" + statsFileName;
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkPlanUtil.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkPlanUtil.java
deleted file mode 100644
index 830d07d86e..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkPlanUtil.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static scala.collection.JavaConverters.seqAsJavaListConverter;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.spark.sql.catalyst.expressions.Expression;
-import org.apache.spark.sql.execution.CommandResultExec;
-import org.apache.spark.sql.execution.SparkPlan;
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper;
-import org.apache.spark.sql.execution.datasources.v2.BatchScanExec;
-import scala.PartialFunction;
-import scala.collection.Seq;
-
-public class SparkPlanUtil {
-
- private static final AdaptiveSparkPlanHelper SPARK_HELPER = new AdaptiveSparkPlanHelper() {};
-
- private SparkPlanUtil() {}
-
- public static List<SparkPlan> collectLeaves(SparkPlan plan) {
- return toJavaList(SPARK_HELPER.collectLeaves(actualPlan(plan)));
- }
-
- public static List<SparkPlan> collectBatchScans(SparkPlan plan) {
- List<SparkPlan> leaves = collectLeaves(plan);
- return leaves.stream()
- .filter(scan -> scan instanceof BatchScanExec)
- .collect(Collectors.toList());
- }
-
- private static SparkPlan actualPlan(SparkPlan plan) {
- if (plan instanceof CommandResultExec) {
- return ((CommandResultExec) plan).commandPhysicalPlan();
- } else {
- return plan;
- }
- }
-
- public static List<Expression> collectExprs(
- SparkPlan sparkPlan, Predicate<Expression> predicate) {
- Seq<List<Expression>> seq =
- SPARK_HELPER.collect(
- sparkPlan,
- new PartialFunction<SparkPlan, List<Expression>>() {
- @Override
- public List<Expression> apply(SparkPlan plan) {
- List<Expression> exprs = Lists.newArrayList();
-
- for (Expression expr : toJavaList(plan.expressions())) {
- exprs.addAll(collectExprs(expr, predicate));
- }
-
- return exprs;
- }
-
- @Override
- public boolean isDefinedAt(SparkPlan plan) {
- return true;
- }
- });
- return toJavaList(seq).stream().flatMap(Collection::stream).collect(Collectors.toList());
- }
-
- private static List<Expression> collectExprs(
- Expression expression, Predicate<Expression> predicate) {
- Seq<Expression> seq =
- expression.collect(
- new PartialFunction<Expression, Expression>() {
- @Override
- public Expression apply(Expression expr) {
- return expr;
- }
-
- @Override
- public boolean isDefinedAt(Expression expr) {
- return predicate.test(expr);
- }
- });
- return toJavaList(seq);
- }
-
- private static <T> List<T> toJavaList(Seq<T> seq) {
- return seqAsJavaListConverter(seq).asJava();
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
deleted file mode 100644
index b5d6415763..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
+++ /dev/null
@@ -1,461 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.apache.iceberg.DataOperations.DELETE;
-import static org.apache.iceberg.DataOperations.OVERWRITE;
-import static org.apache.iceberg.PlanningMode.DISTRIBUTED;
-import static org.apache.iceberg.PlanningMode.LOCAL;
-import static org.apache.iceberg.SnapshotSummary.ADDED_DELETE_FILES_PROP;
-import static org.apache.iceberg.SnapshotSummary.ADDED_DVS_PROP;
-import static org.apache.iceberg.SnapshotSummary.ADDED_FILES_PROP;
-import static org.apache.iceberg.SnapshotSummary.ADD_POS_DELETE_FILES_PROP;
-import static org.apache.iceberg.SnapshotSummary.CHANGED_PARTITION_COUNT_PROP;
-import static org.apache.iceberg.SnapshotSummary.DELETED_FILES_PROP;
-import static org.apache.iceberg.TableProperties.DATA_PLANNING_MODE;
-import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT;
-import static org.apache.iceberg.TableProperties.DELETE_PLANNING_MODE;
-import static org.apache.iceberg.TableProperties.FORMAT_VERSION;
-import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED;
-import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED;
-import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE;
-import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE;
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.stream.Collectors;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.Files;
-import org.apache.iceberg.Parameter;
-import org.apache.iceberg.ParameterizedTestExtension;
-import org.apache.iceberg.Parameters;
-import org.apache.iceberg.PlanningMode;
-import org.apache.iceberg.RowLevelOperationMode;
-import org.apache.iceberg.Snapshot;
-import org.apache.iceberg.SnapshotRef;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.data.GenericRecord;
-import org.apache.iceberg.data.parquet.GenericParquetWriter;
-import org.apache.iceberg.deletes.DeleteGranularity;
-import org.apache.iceberg.io.DataWriter;
-import org.apache.iceberg.io.OutputFile;
-import org.apache.iceberg.parquet.Parquet;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.spark.SparkCatalog;
-import org.apache.iceberg.spark.SparkSessionCatalog;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Encoder;
-import org.apache.spark.sql.Encoders;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.execution.SparkPlan;
-import org.junit.jupiter.api.extension.ExtendWith;
-
-@ExtendWith(ParameterizedTestExtension.class)
-public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase {
-
- private static final Random RANDOM = ThreadLocalRandom.current();
-
- @Parameter(index = 3)
- protected FileFormat fileFormat;
-
- @Parameter(index = 4)
- protected boolean vectorized;
-
- @Parameter(index = 5)
- protected String distributionMode;
-
- @Parameter(index = 6)
- protected boolean fanoutEnabled;
-
- @Parameter(index = 7)
- protected String branch;
-
- @Parameter(index = 8)
- protected PlanningMode planningMode;
-
- @Parameter(index = 9)
- protected int formatVersion;
-
- @Parameters(
- name =
- "catalogName = {0}, implementation = {1}, config = {2},"
- + " format = {3}, vectorized = {4}, distributionMode = {5},"
- + " fanout = {6}, branch = {7}, planningMode = {8}, formatVersion = {9}")
- public static Object[][] parameters() {
- return new Object[][] {
- {
- "testhive",
- SparkCatalog.class.getName(),
- ImmutableMap.of(
- "type", "hive",
- "default-namespace", "default"),
- FileFormat.ORC,
- true,
- WRITE_DISTRIBUTION_MODE_NONE,
- true,
- SnapshotRef.MAIN_BRANCH,
- LOCAL,
- 2
- },
- {
- "testhive",
- SparkCatalog.class.getName(),
- ImmutableMap.of(
- "type", "hive",
- "default-namespace", "default"),
- FileFormat.PARQUET,
- true,
- WRITE_DISTRIBUTION_MODE_NONE,
- false,
- "test",
- DISTRIBUTED,
- 2
- },
- {
- "testhadoop",
- SparkCatalog.class.getName(),
- ImmutableMap.of("type", "hadoop"),
- FileFormat.PARQUET,
- RANDOM.nextBoolean(),
- WRITE_DISTRIBUTION_MODE_HASH,
- true,
- null,
- LOCAL,
- 2
- },
- {
- "spark_catalog",
- SparkSessionCatalog.class.getName(),
- ImmutableMap.of(
- "type", "hive",
- "default-namespace", "default",
- "clients", "1",
- "parquet-enabled", "false",
- "cache-enabled",
- "false" // Spark will delete tables using v1, leaving the cache out of sync
- ),
- FileFormat.AVRO,
- false,
- WRITE_DISTRIBUTION_MODE_RANGE,
- false,
- "test",
- DISTRIBUTED,
- 2
- },
- {
- "testhadoop",
- SparkCatalog.class.getName(),
- ImmutableMap.of("type", "hadoop"),
- FileFormat.PARQUET,
- RANDOM.nextBoolean(),
- WRITE_DISTRIBUTION_MODE_HASH,
- true,
- null,
- LOCAL,
- 3
- },
- {
- "spark_catalog",
- SparkSessionCatalog.class.getName(),
- ImmutableMap.of(
- "type",
- "hive",
- "default-namespace",
- "default",
- "clients",
- "1",
- "parquet-enabled",
- "false",
- "cache-enabled",
- "false" // Spark will delete tables using v1, leaving the cache out of sync
- ),
- FileFormat.AVRO,
- false,
- WRITE_DISTRIBUTION_MODE_RANGE,
- false,
- "test",
- DISTRIBUTED,
- 3
- },
- };
- }
-
- protected abstract Map<String, String> extraTableProperties();
-
- protected void initTable() {
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES('%s' '%s', '%s' '%s', '%s' '%s', '%s' '%s', '%s' '%s', '%s' '%s')",
- tableName,
- DEFAULT_FILE_FORMAT,
- fileFormat,
- WRITE_DISTRIBUTION_MODE,
- distributionMode,
- SPARK_WRITE_PARTITIONED_FANOUT_ENABLED,
- String.valueOf(fanoutEnabled),
- DATA_PLANNING_MODE,
- planningMode.modeName(),
- DELETE_PLANNING_MODE,
- planningMode.modeName(),
- FORMAT_VERSION,
- formatVersion);
-
- switch (fileFormat) {
- case PARQUET:
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES('%s' '%b')",
- tableName, PARQUET_VECTORIZATION_ENABLED, vectorized);
- break;
- case ORC:
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES('%s' '%b')",
- tableName, ORC_VECTORIZATION_ENABLED, vectorized);
- break;
- case AVRO:
- assertThat(vectorized).isFalse();
- break;
- }
-
- Map<String, String> props = extraTableProperties();
- props.forEach(
- (prop, value) -> {
- sql("ALTER TABLE %s SET TBLPROPERTIES('%s' '%s')", tableName, prop, value);
- });
- }
-
- protected void createAndInitTable(String schema) {
- createAndInitTable(schema, null);
- }
-
- protected void createAndInitTable(String schema, String jsonData) {
- createAndInitTable(schema, "", jsonData);
- }
-
- protected void createAndInitTable(String schema, String partitioning, String jsonData) {
- sql("CREATE TABLE %s (%s) USING iceberg %s", tableName, schema, partitioning);
- initTable();
-
- if (jsonData != null) {
- try {
- Dataset<Row> ds = toDS(schema, jsonData);
- ds.coalesce(1).writeTo(tableName).append();
- createBranchIfNeeded();
- } catch (NoSuchTableException e) {
- throw new RuntimeException("Failed to write data", e);
- }
- }
- }
-
- protected void append(String table, String jsonData) {
- append(table, null, jsonData);
- }
-
- protected void append(String table, String schema, String jsonData) {
- try {
- Dataset<Row> ds = toDS(schema, jsonData);
- ds.coalesce(1).writeTo(table).append();
- } catch (NoSuchTableException e) {
- throw new RuntimeException("Failed to write data", e);
- }
- }
-
- protected void createOrReplaceView(String name, String jsonData) {
- createOrReplaceView(name, null, jsonData);
- }
-
- protected void createOrReplaceView(String name, String schema, String jsonData) {
- Dataset<Row> ds = toDS(schema, jsonData);
- ds.createOrReplaceTempView(name);
- }
-
- protected <T> void createOrReplaceView(String name, List<T> data, Encoder<T> encoder) {
- spark.createDataset(data, encoder).createOrReplaceTempView(name);
- }
-
- private Dataset<Row> toDS(String schema, String jsonData) {
- List<String> jsonRows =
- Arrays.stream(jsonData.split("\n"))
- .filter(str -> !str.trim().isEmpty())
- .collect(Collectors.toList());
- Dataset<String> jsonDS = spark.createDataset(jsonRows, Encoders.STRING());
-
- if (schema != null) {
- return spark.read().schema(schema).json(jsonDS);
- } else {
- return spark.read().json(jsonDS);
- }
- }
-
- protected void validateDelete(
- Snapshot snapshot, String changedPartitionCount, String deletedDataFiles) {
- validateSnapshot(snapshot, DELETE, changedPartitionCount, deletedDataFiles, null, null);
- }
-
- protected void validateCopyOnWrite(
- Snapshot snapshot,
- String changedPartitionCount,
- String deletedDataFiles,
- String addedDataFiles) {
- String operation = null == addedDataFiles && null != deletedDataFiles ? DELETE : OVERWRITE;
- validateSnapshot(
- snapshot, operation, changedPartitionCount, deletedDataFiles, null, addedDataFiles);
- }
-
- protected void validateMergeOnRead(
- Snapshot snapshot,
- String changedPartitionCount,
- String addedDeleteFiles,
- String addedDataFiles) {
- String operation = null == addedDataFiles && null != addedDeleteFiles ? DELETE : OVERWRITE;
- validateSnapshot(
- snapshot, operation, changedPartitionCount, null, addedDeleteFiles, addedDataFiles);
- }
-
- protected void validateSnapshot(
- Snapshot snapshot,
- String operation,
- String changedPartitionCount,
- String deletedDataFiles,
- String addedDeleteFiles,
- String addedDataFiles) {
- assertThat(snapshot.operation()).as("Operation must match").isEqualTo(operation);
- validateProperty(snapshot, CHANGED_PARTITION_COUNT_PROP, changedPartitionCount);
- validateProperty(snapshot, DELETED_FILES_PROP, deletedDataFiles);
- validateProperty(snapshot, ADDED_DELETE_FILES_PROP, addedDeleteFiles);
- validateProperty(snapshot, ADDED_FILES_PROP, addedDataFiles);
- if (formatVersion >= 3) {
- validateProperty(snapshot, ADDED_DVS_PROP, addedDeleteFiles);
- assertThat(snapshot.summary()).doesNotContainKey(ADD_POS_DELETE_FILES_PROP);
- }
- }
-
- protected void validateProperty(Snapshot snapshot, String property, Set<String> expectedValues) {
- String actual = snapshot.summary().get(property);
- assertThat(actual)
- .as(
- "Snapshot property "
- + property
- + " has unexpected value, actual = "
- + actual
- + ", expected one of : "
- + String.join(",", expectedValues))
- .isIn(expectedValues);
- }
-
- protected void validateProperty(Snapshot snapshot, String property, String expectedValue) {
- if (null == expectedValue) {
- assertThat(snapshot.summary()).doesNotContainKey(property);
- } else {
- assertThat(snapshot.summary())
- .as("Snapshot property " + property + " has unexpected value.")
- .containsEntry(property, expectedValue);
- }
- }
-
- protected void sleep(long millis) {
- try {
- Thread.sleep(millis);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
- protected DataFile writeDataFile(Table table, List<GenericRecord> records) {
- try {
- OutputFile file =
- Files.localOutput(
- temp.resolve(fileFormat.addExtension(UUID.randomUUID().toString())).toFile());
-
- DataWriter<GenericRecord> dataWriter =
- Parquet.writeData(file)
- .forTable(table)
- .createWriterFunc(GenericParquetWriter::create)
- .overwrite()
- .build();
-
- try {
- for (GenericRecord record : records) {
- dataWriter.write(record);
- }
- } finally {
- dataWriter.close();
- }
-
- return dataWriter.toDataFile();
-
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- @Override
- protected String commitTarget() {
- return branch == null ? tableName : String.format("%s.branch_%s", tableName, branch);
- }
-
- @Override
- protected String selectTarget() {
- return branch == null ? tableName : String.format("%s VERSION AS OF '%s'", tableName, branch);
- }
-
- protected void createBranchIfNeeded() {
- if (branch != null && !branch.equals(SnapshotRef.MAIN_BRANCH)) {
- sql("ALTER TABLE %s CREATE BRANCH %s", tableName, branch);
- }
- }
-
- // ORC currently does not support vectorized reads with deletes
- protected boolean supportsVectorization() {
- return vectorized && (isParquet() || isCopyOnWrite());
- }
-
- private boolean isParquet() {
- return fileFormat.equals(FileFormat.PARQUET);
- }
-
- private boolean isCopyOnWrite() {
- return extraTableProperties().containsValue(RowLevelOperationMode.COPY_ON_WRITE.modeName());
- }
-
- protected void assertAllBatchScansVectorized(SparkPlan plan) {
- List<SparkPlan> batchScans = SparkPlanUtil.collectBatchScans(plan);
- assertThat(batchScans).hasSizeGreaterThan(0).allMatch(SparkPlan::supportsColumnar);
- }
-
- protected void createTableWithDeleteGranularity(
- String schema, String partitionedBy, DeleteGranularity deleteGranularity) {
- createAndInitTable(schema, partitionedBy, null /* empty */);
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')",
- tableName, TableProperties.DELETE_GRANULARITY, deleteGranularity);
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
deleted file mode 100644
index 141de608ea..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
+++ /dev/null
@@ -1,1373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.assertj.core.api.Assumptions.assumeThat;
-
-import java.io.File;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaBuilder;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumWriter;
-import org.apache.iceberg.DataFile;
-import org.apache.iceberg.HasTableOperations;
-import org.apache.iceberg.ManifestFiles;
-import org.apache.iceberg.ManifestReader;
-import org.apache.iceberg.Parameter;
-import org.apache.iceberg.ParameterizedTestExtension;
-import org.apache.iceberg.Parameters;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.spark.Spark3Util;
-import org.apache.iceberg.spark.SparkCatalogConfig;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.apache.spark.sql.catalyst.parser.ParseException;
-import org.apache.spark.sql.types.DataTypes;
-import org.apache.spark.sql.types.Metadata;
-import org.apache.spark.sql.types.StructField;
-import org.apache.spark.sql.types.StructType;
-import org.joda.time.DateTime;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.extension.ExtendWith;
-
-@ExtendWith(ParameterizedTestExtension.class)
-public class TestAddFilesProcedure extends ExtensionsTestBase {
-
- @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}, formatVersion = {3}")
- public static Object[][] parameters() {
- return new Object[][] {
- {
- SparkCatalogConfig.HIVE.catalogName(),
- SparkCatalogConfig.HIVE.implementation(),
- SparkCatalogConfig.HIVE.properties(),
- 1
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
- 2
- },
- {
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
- 2
- }
- };
- }
-
- @Parameter(index = 3)
- private int formatVersion;
-
- private final String sourceTableName = "source_table";
- private File fileTableDir;
-
- @BeforeEach
- public void setupTempDirs() {
- fileTableDir = temp.toFile();
- }
-
- @AfterEach
- public void dropTables() {
- sql("DROP TABLE IF EXISTS %s PURGE", sourceTableName);
- sql("DROP TABLE IF EXISTS %s", tableName);
- }
-
- @TestTemplate
- public void addDataUnpartitioned() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void deleteAndAddBackUnpartitioned() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- String deleteData = "DELETE FROM %s";
- sql(deleteData, tableName);
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @Disabled // TODO Classpath issues prevent us from actually writing to a Spark ORC table
- public void addDataUnpartitionedOrc() {
- createUnpartitionedFileTable("orc");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`orc`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertThat(result).isEqualTo(2L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addAvroFile() throws Exception {
- // Spark Session Catalog cannot load metadata tables
- // with "The namespace in session catalog must have exactly one name part"
- assumeThat(catalogName).isNotEqualTo("spark_catalog");
-
- // Create an Avro file
-
- Schema schema =
- SchemaBuilder.record("record")
- .fields()
- .requiredInt("id")
- .requiredString("data")
- .endRecord();
- GenericRecord record1 = new GenericData.Record(schema);
- record1.put("id", 1L);
- record1.put("data", "a");
- GenericRecord record2 = new GenericData.Record(schema);
- record2.put("id", 2L);
- record2.put("data", "b");
- File outputFile = temp.resolve("test.avro").toFile();
-
- DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter(schema);
- DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter(datumWriter);
- dataFileWriter.create(schema, outputFile);
- dataFileWriter.append(record1);
- dataFileWriter.append(record2);
- dataFileWriter.close();
-
- createIcebergTable("id Long, data String");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`avro`.`%s`')",
- catalogName, tableName, outputFile.getPath());
- assertOutput(result, 1L, 1L);
-
- List<Object[]> expected = Lists.newArrayList(new Object[] {1L, "a"}, new Object[] {2L, "b"});
-
- assertEquals(
- "Iceberg table contains correct data",
- expected,
- sql("SELECT * FROM %s ORDER BY id", tableName));
-
- List<Object[]> actualRecordCount =
- sql("select %s from %s.files", DataFile.RECORD_COUNT.name(), tableName);
- List<Object[]> expectedRecordCount = Lists.newArrayList();
- expectedRecordCount.add(new Object[] {2L});
- assertEquals(
- "Iceberg file metadata should have correct metadata count",
- expectedRecordCount,
- actualRecordCount);
- }
-
- // TODO Adding spark-avro doesn't work in tests
- @Disabled
- public void addDataUnpartitionedAvro() {
- createUnpartitionedFileTable("avro");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`avro`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertThat(result).isEqualTo(2L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addDataUnpartitionedHive() {
- createUnpartitionedHiveTable();
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- List<Object[]> result =
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addDataUnpartitionedExtraCol() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String, foo string");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addDataUnpartitionedMissingCol() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addDataPartitionedMissingCol() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 8L, 4L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addDataPartitioned() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 8L, 4L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @Disabled // TODO Classpath issues prevent us from actually writing to a Spark ORC table
- public void addDataPartitionedOrc() {
- createPartitionedFileTable("orc");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertThat(result).isEqualTo(8L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- // TODO Adding spark-avro doesn't work in tests
- @Disabled
- public void addDataPartitionedAvro() {
- createPartitionedFileTable("avro");
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg PARTITIONED BY (id)";
-
- sql(createIceberg, tableName);
-
- Object result =
- scalarSql(
- "CALL %s.system.add_files('%s', '`avro`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertThat(result).isEqualTo(8L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addDataPartitionedHive() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
-
- assertOutput(result, 8L, 4L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addPartitionToPartitioned() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void deleteAndAddBackPartitioned() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- String deleteData = "DELETE FROM %s where id = 1";
- sql(deleteData, tableName);
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addPartitionToPartitionedSnapshotIdInheritanceEnabledInTwoRuns()
- throws NoSuchTableException, ParseException {
- createPartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')",
- tableName, TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED);
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 2))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id < 3 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
-
- manifestSpecMatchesTableSpec();
-
- verifyUUIDInPath();
- }
-
- @TestTemplate
- public void addPartitionsFromHiveSnapshotInheritanceEnabled()
- throws NoSuchTableException, ParseException {
- createPartitionedHiveTable();
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- sql(
- "ALTER TABLE %s SET TBLPROPERTIES ('%s' 'true')",
- tableName, TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED);
-
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
-
- manifestSpecMatchesTableSpec();
-
- verifyUUIDInPath();
- }
-
- @TestTemplate
- public void addDataPartitionedByDateToPartitioned() {
- createDatePartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, date Date", "PARTITIONED BY (date)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('date', '2021-01-01'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, date FROM %s WHERE date = '2021-01-01' ORDER BY id", sourceTableName),
- sql("SELECT id, name, date FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addDataPartitionedVerifyPartitionTypeInferredCorrectly() {
- createTableWithTwoPartitions("parquet");
-
- createIcebergTable(
- "id Integer, name String, date Date, dept String", "PARTITIONED BY (date, dept)");
-
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('date', '2021-01-01'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- String sqlFormat =
- "SELECT id, name, dept, date FROM %s WHERE date = '2021-01-01' and dept= '01' ORDER BY id";
- assertEquals(
- "Iceberg table contains correct data",
- sql(sqlFormat, sourceTableName),
- sql(sqlFormat, tableName));
- }
-
- @TestTemplate
- public void addFilteredPartitionsToPartitioned() {
- createCompositePartitionedTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addFilteredPartitionsToPartitioned2() {
- createCompositePartitionedTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 6L, 3L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql(
- "SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id",
- sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnId() {
- createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addFilteredPartitionsToPartitionedWithNullValueFilteringOnDept() {
- createCompositePartitionedTableWithNullValueInPartitionColumn("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id, dept)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', 'hr'))",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 6L, 3L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql(
- "SELECT id, name, dept, subdept FROM %s WHERE dept = 'hr' ORDER BY id",
- sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addFileTableOldSpecDataAfterPartitionSpecEvolved()
- throws NoSuchTableException, ParseException {
- createPartitionedFileTable("parquet");
- createIcebergTable(
- "id Integer, name String, dept String, subdept String",
- "PARTITIONED BY (id, dept, subdept)");
- sql("ALTER TABLE %s DROP PARTITION FIELD dept", tableName);
- sql(
- "ALTER TABLE %s DROP PARTITION FIELD subdept",
- tableName); // This spec now matches the partitioning of the parquet table
- sql("ALTER TABLE %s ADD PARTITION FIELD subdept", tableName);
-
- if (formatVersion == 1) {
- // In V1, since we are dropping the partition field, it adds a void transform which will not
- // match with the input spec
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining(
- String.format(
- "Cannot find a partition spec in Iceberg table %s that matches the partition columns ([id]) in input table",
- tableName));
- return;
- }
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files(table => '%s', source_table => '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
-
- assertOutput(result, 8L, 4L);
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
-
- Table table = Spark3Util.loadIcebergTable(spark, tableName);
- // Find the spec that matches the partitioning of the parquet table
- PartitionSpec compatibleSpec =
- table.specs().values().stream()
- .filter(spec -> spec.fields().size() == 1)
- .filter(spec -> "id".equals(spec.fields().get(0).name()))
- .findFirst()
- .orElse(null);
-
- assertThat(compatibleSpec).isNotNull();
- manifestSpecMatchesGivenSpec(table, compatibleSpec);
- verifyUUIDInPath();
- }
-
- @TestTemplate
- public void addFileTableNoCompatibleSpec() {
- createPartitionedFileTable("parquet");
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (dept)");
- sql("ALTER TABLE %s ADD PARTITION FIELD subdept", tableName);
-
- String fullTableName = tableName;
- if (implementation.equals(SparkCatalogConfig.SPARK.implementation())) {
- fullTableName = String.format("%s.%s", catalogName, tableName);
- }
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files(table => '%s', source_table => '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining(
- String.format(
- "Cannot find a partition spec in Iceberg table %s that matches the partition columns (%s) in input table",
- fullTableName, "[id]"));
- }
-
- @TestTemplate
- public void addWeirdCaseHiveTable() {
- createWeirdCaseTable();
-
- createIcebergTable(
- "id Integer, `naMe` String, dept String, subdept String", "PARTITIONED BY (`naMe`)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '%s', map('naMe', 'John Doe'))",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result, 2L, 1L);
-
- /*
- While we would like to use
- SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id
- Spark does not actually handle this pushdown correctly for hive based tables and it returns 0 records
- */
- List<Object[]> expected =
- sql("SELECT id, `naMe`, dept, subdept from %s ORDER BY id", sourceTableName).stream()
- .filter(r -> r[1].equals("John Doe"))
- .collect(Collectors.toList());
-
- // TODO when this assert breaks Spark fixed the pushdown issue
- assertThat(
- sql(
- "SELECT id, `naMe`, dept, subdept from %s WHERE `naMe` = 'John Doe' ORDER BY id",
- sourceTableName))
- .as("If this assert breaks it means that Spark has fixed the pushdown issue")
- .isEmpty();
-
- // Pushdown works for iceberg
- assertThat(
- sql(
- "SELECT id, `naMe`, dept, subdept FROM %s WHERE `naMe` = 'John Doe' ORDER BY id",
- tableName))
- .as("We should be able to pushdown mixed case partition keys")
- .hasSize(2);
-
- assertEquals(
- "Iceberg table contains correct data",
- expected,
- sql("SELECT id, `naMe`, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void addPartitionToPartitionedHive() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '%s', map('id', 1))",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void invalidDataImport() {
- createPartitionedFileTable("parquet");
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('id', 1))",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
- .hasMessageContaining("that matches the partition columns");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
- .hasMessageContaining("that matches the partition columns");
- }
-
- @TestTemplate
- public void invalidDataImportPartitioned() {
- createUnpartitionedFileTable("parquet");
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('x', '1', 'y', '2'))",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
- .hasMessageContaining("that matches the partition columns");
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`', map('dept', '2'))",
- catalogName, tableName, fileTableDir.getAbsolutePath()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageStartingWith("Cannot find a partition spec in Iceberg table")
- .hasMessageContaining("that matches the partition columns");
- }
-
- @TestTemplate
- public void partitionColumnCountMismatchInFilter() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '%s', map('id', '0', 'dept', '1'))",
- catalogName, tableName, sourceTableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageStartingWith("Cannot add data files to target table")
- .hasMessageContaining(
- "because that table is partitioned, but the number of columns in the provided partition filter (2)"
- + " is greater than the number of partitioned columns in table (1)");
- }
-
- @TestTemplate
- public void invalidPartitionColumnsInFilter() {
- createPartitionedHiveTable();
-
- String icebergTablePartitionNames = "id";
- createIcebergTable(
- "id Integer, name String, dept String, subdept String",
- String.format("PARTITIONED BY (%s)", icebergTablePartitionNames));
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '%s', map('dept', '1'))",
- catalogName, tableName, sourceTableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageStartingWith("Cannot add files to target table")
- .hasMessageContaining(
- "specified partition filter refers to columns that are not partitioned: [dept]")
- .hasMessageContaining("Valid partition columns: [%s]", icebergTablePartitionNames);
- }
-
- @TestTemplate
- public void addTwice() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result1 =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1))",
- catalogName, tableName, sourceTableName);
- assertOutput(result1, 2L, 1L);
-
- List<Object[]> result2 =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 2))",
- catalogName, tableName, sourceTableName);
- assertOutput(result2, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", tableName));
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 2 ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void duplicateDataPartitioned() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1))",
- catalogName, tableName, sourceTableName);
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1))",
- catalogName, tableName, sourceTableName))
- .isInstanceOf(IllegalStateException.class)
- .hasMessageStartingWith(
- "Cannot complete import because data files to be imported already"
- + " exist within the target table");
- }
-
- @TestTemplate
- public void duplicateDataPartitionedAllowed() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result1 =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1))",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result1, 2L, 1L);
-
- List<Object[]> result2 =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', 1),"
- + "check_duplicate_files => false)",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result2, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql(
- "SELECT id, name, dept, subdept FROM %s WHERE id = 1 UNION ALL "
- + "SELECT id, name, dept, subdept FROM %s WHERE id = 1",
- sourceTableName, sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s", tableName, tableName));
- }
-
- @TestTemplate
- public void duplicateDataUnpartitioned() {
- createUnpartitionedHiveTable();
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
-
- assertThatThrownBy(
- () ->
- scalarSql(
- "CALL %s.system.add_files('%s', '%s')",
- catalogName, tableName, sourceTableName))
- .isInstanceOf(IllegalStateException.class)
- .hasMessageStartingWith(
- "Cannot complete import because data files to be imported already"
- + " exist within the target table");
- }
-
- @TestTemplate
- public void duplicateDataUnpartitionedAllowed() {
- createUnpartitionedHiveTable();
-
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- List<Object[]> result1 =
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
- assertOutput(result1, 2L, 1L);
-
- List<Object[]> result2 =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s',"
- + "check_duplicate_files => false)",
- catalogName, tableName, sourceTableName);
- assertOutput(result2, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql(
- "SELECT * FROM (SELECT * FROM %s UNION ALL " + "SELECT * from %s) ORDER BY id",
- sourceTableName, sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void testEmptyImportDoesNotThrow() {
- createIcebergTable("id Integer, name String, dept String, subdept String");
-
- // Empty path based import
- List<Object[]> pathResult =
- sql(
- "CALL %s.system.add_files('%s', '`parquet`.`%s`')",
- catalogName, tableName, fileTableDir.getAbsolutePath());
- assertOutput(pathResult, 0L, 0L);
- assertEquals(
- "Iceberg table contains no added data when importing from an empty path",
- EMPTY_QUERY_RESULT,
- sql("SELECT * FROM %s ORDER BY id", tableName));
-
- // Empty table based import
- String createHive =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) STORED AS parquet";
- sql(createHive, sourceTableName);
-
- List<Object[]> tableResult =
- sql("CALL %s.system.add_files('%s', '%s')", catalogName, tableName, sourceTableName);
- assertOutput(tableResult, 0L, 0L);
- assertEquals(
- "Iceberg table contains no added data when importing from an empty table",
- EMPTY_QUERY_RESULT,
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void testPartitionedImportFromEmptyPartitionDoesNotThrow() {
- createPartitionedHiveTable();
-
- final int emptyPartitionId = 999;
- // Add an empty partition to the hive table
- sql(
- "ALTER TABLE %s ADD PARTITION (id = '%d') LOCATION '%d'",
- sourceTableName, emptyPartitionId, emptyPartitionId);
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> tableResult =
- sql(
- "CALL %s.system.add_files("
- + "table => '%s', "
- + "source_table => '%s', "
- + "partition_filter => map('id', %d))",
- catalogName, tableName, sourceTableName, emptyPartitionId);
-
- assertOutput(tableResult, 0L, 0L);
- assertEquals(
- "Iceberg table contains no added data when importing from an empty table",
- EMPTY_QUERY_RESULT,
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void testAddFilesWithParallelism() {
- createUnpartitionedHiveTable();
-
- String createIceberg =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING iceberg";
-
- sql(createIceberg, tableName);
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files(table => '%s', source_table => '%s', parallelism => 2)",
- catalogName, tableName, sourceTableName);
-
- assertEquals("Procedure output must match", ImmutableList.of(row(2L, 1L)), result);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT * FROM %s ORDER BY id", sourceTableName),
- sql("SELECT * FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void testAddFilesPartitionedWithParallelism() {
- createPartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files(table => '%s', source_table => '%s', parallelism => 2)",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result, 8L, 4L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- @TestTemplate
- public void testAddFilesWithInvalidParallelism() {
- createUnpartitionedHiveTable();
-
- createIcebergTable(
- "id Integer, name String, dept String, subdept String", "PARTITIONED BY (id)");
-
- assertThatThrownBy(
- () ->
- sql(
- "CALL %s.system.add_files(table => '%s', source_table => '%s', parallelism => -1)",
- catalogName, tableName, sourceTableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Parallelism should be larger than 0");
- }
-
- @TestTemplate
- public void testAddFilesToTableWithManySpecs() {
- createPartitionedHiveTable();
- createIcebergTable("id Integer, name String, dept String, subdept String"); // Spec 0
-
- sql("ALTER TABLE %s ADD PARTITION FIELD id", tableName); // Spec 1
- sql("ALTER TABLE %s ADD PARTITION FIELD name", tableName); // Spec 2
- sql("ALTER TABLE %s ADD PARTITION FIELD subdept", tableName); // Spec 3
-
- List<Object[]> result =
- sql(
- "CALL %s.system.add_files('%s', '%s', map('id', 1))",
- catalogName, tableName, sourceTableName);
-
- assertOutput(result, 2L, 1L);
-
- assertEquals(
- "Iceberg table contains correct data",
- sql("SELECT id, name, dept, subdept FROM %s WHERE id = 1 ORDER BY id", sourceTableName),
- sql("SELECT id, name, dept, subdept FROM %s ORDER BY id", tableName));
- }
-
- private static final List<Object[]> EMPTY_QUERY_RESULT = Lists.newArrayList();
-
- private static final StructField[] STRUCT = {
- new StructField("id", DataTypes.IntegerType, true, Metadata.empty()),
- new StructField("name", DataTypes.StringType, true, Metadata.empty()),
- new StructField("dept", DataTypes.StringType, true, Metadata.empty()),
- new StructField("subdept", DataTypes.StringType, true, Metadata.empty())
- };
-
- private Dataset<Row> unpartitionedDF() {
- return spark
- .createDataFrame(
- ImmutableList.of(
- RowFactory.create(1, "John Doe", "hr", "communications"),
- RowFactory.create(2, "Jane Doe", "hr", "salary"),
- RowFactory.create(3, "Matt Doe", "hr", "communications"),
- RowFactory.create(4, "Will Doe", "facilities", "all")),
- new StructType(STRUCT))
- .repartition(1);
- }
-
- private Dataset<Row> singleNullRecordDF() {
- return spark
- .createDataFrame(
- ImmutableList.of(RowFactory.create(null, null, null, null)), new StructType(STRUCT))
- .repartition(1);
- }
-
- private Dataset<Row> partitionedDF() {
- return unpartitionedDF().select("name", "dept", "subdept", "id");
- }
-
- private Dataset<Row> compositePartitionedDF() {
- return unpartitionedDF().select("name", "subdept", "id", "dept");
- }
-
- private Dataset<Row> compositePartitionedNullRecordDF() {
- return singleNullRecordDF().select("name", "subdept", "id", "dept");
- }
-
- private Dataset<Row> weirdColumnNamesDF() {
- Dataset<Row> unpartitionedDF = unpartitionedDF();
- return unpartitionedDF.select(
- unpartitionedDF.col("id"),
- unpartitionedDF.col("subdept"),
- unpartitionedDF.col("dept"),
- unpartitionedDF.col("name").as("naMe"));
- }
-
- private static final StructField[] DATE_STRUCT = {
- new StructField("id", DataTypes.IntegerType, true, Metadata.empty()),
- new StructField("name", DataTypes.StringType, true, Metadata.empty()),
- new StructField("ts", DataTypes.DateType, true, Metadata.empty()),
- new StructField("dept", DataTypes.StringType, true, Metadata.empty()),
- };
-
- private static java.sql.Date toDate(String value) {
- return new java.sql.Date(DateTime.parse(value).getMillis());
- }
-
- private Dataset<Row> dateDF() {
- return spark
- .createDataFrame(
- ImmutableList.of(
- RowFactory.create(1, "John Doe", toDate("2021-01-01"), "01"),
- RowFactory.create(2, "Jane Doe", toDate("2021-01-01"), "01"),
- RowFactory.create(3, "Matt Doe", toDate("2021-01-02"), "02"),
- RowFactory.create(4, "Will Doe", toDate("2021-01-02"), "02")),
- new StructType(DATE_STRUCT))
- .repartition(2);
- }
-
- private void createUnpartitionedFileTable(String format) {
- String createParquet =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s LOCATION '%s'";
-
- sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
- Dataset<Row> df = unpartitionedDF();
- df.write().insertInto(sourceTableName);
- df.write().insertInto(sourceTableName);
- }
-
- private void createPartitionedFileTable(String format) {
- String createParquet =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s PARTITIONED BY (id) "
- + "LOCATION '%s'";
-
- sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
-
- Dataset<Row> df = partitionedDF();
- df.write().insertInto(sourceTableName);
- df.write().insertInto(sourceTableName);
- }
-
- private void createCompositePartitionedTable(String format) {
- String createParquet =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s "
- + "PARTITIONED BY (id, dept) LOCATION '%s'";
- sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
-
- Dataset<Row> df = compositePartitionedDF();
- df.write().insertInto(sourceTableName);
- df.write().insertInto(sourceTableName);
- }
-
- private void createCompositePartitionedTableWithNullValueInPartitionColumn(String format) {
- String createParquet =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) USING %s "
- + "PARTITIONED BY (id, dept) LOCATION '%s'";
- sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
-
- Dataset<Row> unionedDF =
- compositePartitionedDF()
- .unionAll(compositePartitionedNullRecordDF())
- .select("name", "subdept", "id", "dept")
- .repartition(1);
-
- unionedDF.write().insertInto(sourceTableName);
- unionedDF.write().insertInto(sourceTableName);
- }
-
- private void createWeirdCaseTable() {
- String createParquet =
- "CREATE TABLE %s (id Integer, subdept String, dept String) "
- + "PARTITIONED BY (`naMe` String) STORED AS parquet";
-
- sql(createParquet, sourceTableName);
-
- Dataset<Row> df = weirdColumnNamesDF();
- df.write().insertInto(sourceTableName);
- df.write().insertInto(sourceTableName);
- }
-
- private void createUnpartitionedHiveTable() {
- String createHive =
- "CREATE TABLE %s (id Integer, name String, dept String, subdept String) STORED AS parquet";
-
- sql(createHive, sourceTableName);
-
- Dataset<Row> df = unpartitionedDF();
- df.write().insertInto(sourceTableName);
- df.write().insertInto(sourceTableName);
- }
-
- private void createPartitionedHiveTable() {
- String createHive =
- "CREATE TABLE %s (name String, dept String, subdept String) "
- + "PARTITIONED BY (id Integer) STORED AS parquet";
-
- sql(createHive, sourceTableName);
-
- Dataset<Row> df = partitionedDF();
- df.write().insertInto(sourceTableName);
- df.write().insertInto(sourceTableName);
- }
-
- private void createDatePartitionedFileTable(String format) {
- String createParquet =
- "CREATE TABLE %s (id Integer, name String, date Date) USING %s "
- + "PARTITIONED BY (date) LOCATION '%s'";
-
- sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
-
- dateDF().select("id", "name", "ts").write().insertInto(sourceTableName);
- }
-
- private void createTableWithTwoPartitions(String format) {
- String createParquet =
- "CREATE TABLE %s (id Integer, name String, date Date, dept String) USING %s "
- + "PARTITIONED BY (date, dept) LOCATION '%s'";
-
- sql(createParquet, sourceTableName, format, fileTableDir.getAbsolutePath());
-
- dateDF().write().insertInto(sourceTableName);
- }
-
- private void createIcebergTable(String schema) {
- createIcebergTable(schema, "");
- }
-
- private void createIcebergTable(String schema, String partitioning) {
- sql(
- "CREATE TABLE %s (%s) USING iceberg %s TBLPROPERTIES ('%s' '%d')",
- tableName, schema, partitioning, TableProperties.FORMAT_VERSION, formatVersion);
- }
-
- private void assertOutput(
- List<Object[]> result, long expectedAddedFilesCount, long expectedChangedPartitionCount) {
- Object[] output = Iterables.getOnlyElement(result);
- assertThat(output[0]).isEqualTo(expectedAddedFilesCount);
- if (formatVersion == 1) {
- assertThat(output[1]).isEqualTo(expectedChangedPartitionCount);
- } else {
- // the number of changed partitions may not be populated in v2 tables
- assertThat(output[1]).isIn(expectedChangedPartitionCount, null);
- }
- }
-
- private void manifestSpecMatchesTableSpec() throws NoSuchTableException, ParseException {
- Table table = Spark3Util.loadIcebergTable(spark, tableName);
- manifestSpecMatchesGivenSpec(table, table.spec());
- }
-
- private void manifestSpecMatchesGivenSpec(Table table, PartitionSpec partitionSpec) {
- FileIO io = ((HasTableOperations) table).operations().io();
- // Check that the manifests have the correct partition spec
- assertThat(
- table.currentSnapshot().allManifests(io).stream()
- .map(mf -> ManifestFiles.read(mf, io, null /* force reading spec from file*/))
- .map(ManifestReader::spec)
- .collect(Collectors.toList()))
- .allSatisfy(spec -> assertThat(spec).isEqualTo(partitionSpec));
- }
-
- private void verifyUUIDInPath() {
- // verify manifest file name has uuid pattern
- String manifestPath = (String) sql("select path from %s.manifests", tableName).get(0)[0];
-
- Pattern uuidPattern = Pattern.compile("[a-f0-9]{8}(?:-[a-f0-9]{4}){4}[a-f0-9]{8}");
-
- Matcher matcher = uuidPattern.matcher(manifestPath);
- assertThat(matcher.find()).as("verify manifest path has uuid").isTrue();
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
deleted file mode 100644
index dd49d8a254..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
+++ /dev/null
@@ -1,586 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import org.apache.iceberg.Parameter;
-import org.apache.iceberg.ParameterizedTestExtension;
-import org.apache.iceberg.Parameters;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.TestHelpers;
-import org.apache.iceberg.spark.SparkCatalogConfig;
-import org.apache.iceberg.spark.source.SparkTable;
-import org.apache.spark.sql.connector.catalog.CatalogManager;
-import org.apache.spark.sql.connector.catalog.Identifier;
-import org.apache.spark.sql.connector.catalog.TableCatalog;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.extension.ExtendWith;
-
-@ExtendWith(ParameterizedTestExtension.class)
-public class TestAlterTablePartitionFields extends ExtensionsTestBase {
-
- @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}, formatVersion = {3}")
- public static Object[][] parameters() {
- return new Object[][] {
- {
- SparkCatalogConfig.HIVE.catalogName(),
- SparkCatalogConfig.HIVE.implementation(),
- SparkCatalogConfig.HIVE.properties(),
- 1
- },
- {
- SparkCatalogConfig.SPARK.catalogName(),
- SparkCatalogConfig.SPARK.implementation(),
- SparkCatalogConfig.SPARK.properties(),
- 2
- }
- };
- }
-
- @Parameter(index = 3)
- private int formatVersion;
-
- @AfterEach
- public void removeTable() {
- sql("DROP TABLE IF EXISTS %s", tableName);
- }
-
- @TestTemplate
- public void testAddIdentityPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD category", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).identity("category").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddBucketPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(1)
- .bucket("id", 16, "id_bucket_16")
- .build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddTruncatePartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD truncate(data, 4)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(1)
- .truncate("data", 4, "data_trunc_4")
- .build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddYearsPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddMonthsPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD months(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddDaysPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddHoursPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD hours(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddYearPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddMonthPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddDayPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddHourPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testAddNamedPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
-
- table.refresh();
-
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).bucket("id", 16, "shard").build();
-
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- }
-
- @TestTemplate
- public void testDropIdentityPartition() {
- createTable("id bigint NOT NULL, category string, data string", "category");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().fields()).as("Table should start with 1 partition field").hasSize(1);
-
- sql("ALTER TABLE %s DROP PARTITION FIELD category", tableName);
-
- table.refresh();
-
- if (formatVersion == 1) {
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(1)
- .alwaysNull("category", "category")
- .build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- } else {
- assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
- }
- }
-
- @TestTemplate
- public void testDropDaysPartition() {
- createTable("id bigint NOT NULL, ts timestamp, data string", "days(ts)");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().fields()).as("Table should start with 1 partition field").hasSize(1);
-
- sql("ALTER TABLE %s DROP PARTITION FIELD days(ts)", tableName);
-
- table.refresh();
-
- if (formatVersion == 1) {
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).alwaysNull("ts", "ts_day").build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- } else {
- assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
- }
- }
-
- @TestTemplate
- public void testDropBucketPartition() {
- createTable("id bigint NOT NULL, data string", "bucket(16, id)");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().fields()).as("Table should start with 1 partition field").hasSize(1);
-
- sql("ALTER TABLE %s DROP PARTITION FIELD bucket(16, id)", tableName);
-
- table.refresh();
-
- if (formatVersion == 1) {
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(1)
- .alwaysNull("id", "id_bucket")
- .build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- } else {
- assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
- }
- }
-
- @TestTemplate
- public void testDropPartitionByName() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
-
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
-
- table.refresh();
-
- assertThat(table.spec().fields()).as("Table should have 1 partition field").hasSize(1);
-
- // Should be recognized as iceberg command even with extra white spaces
- sql("ALTER TABLE %s DROP PARTITION \n FIELD shard", tableName);
-
- table.refresh();
-
- if (formatVersion == 1) {
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(2).alwaysNull("id", "shard").build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
- } else {
- assertThat(table.spec().isUnpartitioned()).as("New spec must be unpartitioned").isTrue();
- }
- }
-
- @TestTemplate
- public void testReplacePartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
- table.refresh();
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
-
- sql("ALTER TABLE %s REPLACE PARTITION FIELD days(ts) WITH hours(ts)", tableName);
- table.refresh();
- if (formatVersion == 1) {
- expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(2)
- .alwaysNull("ts", "ts_day")
- .hour("ts")
- .build();
- } else {
- expected =
- TestHelpers.newExpectedSpecBuilder()
- .withSchema(table.schema())
- .withSpecId(2)
- .addField("hour", 3, 1001, "ts_hour")
- .build();
- }
- assertThat(table.spec())
- .as("Should changed from daily to hourly partitioned field")
- .isEqualTo(expected);
- }
-
- @TestTemplate
- public void testReplacePartitionAndRename() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD days(ts)", tableName);
- table.refresh();
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
-
- sql("ALTER TABLE %s REPLACE PARTITION FIELD days(ts) WITH hours(ts) AS hour_col", tableName);
- table.refresh();
- if (formatVersion == 1) {
- expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(2)
- .alwaysNull("ts", "ts_day")
- .hour("ts", "hour_col")
- .build();
- } else {
- expected =
- TestHelpers.newExpectedSpecBuilder()
- .withSchema(table.schema())
- .withSpecId(2)
- .addField("hour", 3, 1001, "hour_col")
- .build();
- }
- assertThat(table.spec())
- .as("Should changed from daily to hourly partitioned field")
- .isEqualTo(expected);
- }
-
- @TestTemplate
- public void testReplaceNamedPartition() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD days(ts) AS day_col", tableName);
- table.refresh();
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts", "day_col").build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
-
- sql("ALTER TABLE %s REPLACE PARTITION FIELD day_col WITH hours(ts)", tableName);
- table.refresh();
- if (formatVersion == 1) {
- expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(2)
- .alwaysNull("ts", "day_col")
- .hour("ts")
- .build();
- } else {
- expected =
- TestHelpers.newExpectedSpecBuilder()
- .withSchema(table.schema())
- .withSpecId(2)
- .addField("hour", 3, 1001, "ts_hour")
- .build();
- }
- assertThat(table.spec())
- .as("Should changed from daily to hourly partitioned field")
- .isEqualTo(expected);
- }
-
- @TestTemplate
- public void testReplaceNamedPartitionAndRenameDifferently() {
- createTable("id bigint NOT NULL, category string, ts timestamp, data string");
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.spec().isUnpartitioned()).as("Table should start unpartitioned").isTrue();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD days(ts) AS day_col", tableName);
- table.refresh();
- PartitionSpec expected =
- PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts", "day_col").build();
- assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected);
-
- sql("ALTER TABLE %s REPLACE PARTITION FIELD day_col WITH hours(ts) AS hour_col", tableName);
- table.refresh();
- if (formatVersion == 1) {
- expected =
- PartitionSpec.builderFor(table.schema())
- .withSpecId(2)
- .alwaysNull("ts", "day_col")
- .hour("ts", "hour_col")
- .build();
- } else {
- expected =
- TestHelpers.newExpectedSpecBuilder()
- .withSchema(table.schema())
- .withSpecId(2)
- .addField("hour", 3, 1001, "hour_col")
- .build();
- }
- assertThat(table.spec())
- .as("Should changed from daily to hourly partitioned field")
- .isEqualTo(expected);
- }
-
- @TestTemplate
- public void testSparkTableAddDropPartitions() throws Exception {
- createTable("id bigint NOT NULL, ts timestamp, data string");
- assertThat(sparkTable().partitioning()).as("spark table partition should be empty").isEmpty();
-
- sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, id) AS shard", tableName);
- assertPartitioningEquals(sparkTable(), 1, "bucket(16, id)");
-
- sql("ALTER TABLE %s ADD PARTITION FIELD truncate(data, 4)", tableName);
- assertPartitioningEquals(sparkTable(), 2, "truncate(4, data)");
-
- sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName);
- assertPartitioningEquals(sparkTable(), 3, "years(ts)");
-
- sql("ALTER TABLE %s DROP PARTITION FIELD years(ts)", tableName);
- assertPartitioningEquals(sparkTable(), 2, "truncate(4, data)");
-
- sql("ALTER TABLE %s DROP PARTITION FIELD truncate(4, data)", tableName);
- assertPartitioningEquals(sparkTable(), 1, "bucket(16, id)");
-
- sql("ALTER TABLE %s DROP PARTITION FIELD shard", tableName);
- sql("DESCRIBE %s", tableName);
- assertThat(sparkTable().partitioning()).as("spark table partition should be empty").isEmpty();
- }
-
- @TestTemplate
- public void testDropColumnOfOldPartitionFieldV1() {
- // default table created in v1 format
- sql(
- "CREATE TABLE %s (id bigint NOT NULL, ts timestamp, day_of_ts date) USING iceberg PARTITIONED BY (day_of_ts) TBLPROPERTIES('format-version' = '1')",
- tableName);
-
- sql("ALTER TABLE %s REPLACE PARTITION FIELD day_of_ts WITH days(ts)", tableName);
-
- sql("ALTER TABLE %s DROP COLUMN day_of_ts", tableName);
- }
-
- @TestTemplate
- public void testDropColumnOfOldPartitionFieldV2() {
- sql(
- "CREATE TABLE %s (id bigint NOT NULL, ts timestamp, day_of_ts date) USING iceberg PARTITIONED BY (day_of_ts) TBLPROPERTIES('format-version' = '2')",
- tableName);
-
- sql("ALTER TABLE %s REPLACE PARTITION FIELD day_of_ts WITH days(ts)", tableName);
-
- sql("ALTER TABLE %s DROP COLUMN day_of_ts", tableName);
- }
-
- private void assertPartitioningEquals(SparkTable table, int len, String transform) {
- assertThat(table.partitioning()).as("spark table partition should be " + len).hasSize(len);
- assertThat(table.partitioning()[len - 1])
- .asString()
- .as("latest spark table partition transform should match")
- .isEqualTo(transform);
- }
-
- private SparkTable sparkTable() throws Exception {
- validationCatalog.loadTable(tableIdent).refresh();
- CatalogManager catalogManager = spark.sessionState().catalogManager();
- TableCatalog catalog = (TableCatalog) catalogManager.catalog(catalogName);
- Identifier identifier = Identifier.of(tableIdent.namespace().levels(), tableIdent.name());
- return (SparkTable) catalog.loadTable(identifier);
- }
-
- private void createTable(String schema) {
- createTable(schema, null);
- }
-
- private void createTable(String schema, String spec) {
- if (spec == null) {
- sql(
- "CREATE TABLE %s (%s) USING iceberg TBLPROPERTIES ('%s' '%d')",
- tableName, schema, TableProperties.FORMAT_VERSION, formatVersion);
- } else {
- sql(
- "CREATE TABLE %s (%s) USING iceberg PARTITIONED BY (%s) TBLPROPERTIES ('%s' '%d')",
- tableName, schema, spec, TableProperties.FORMAT_VERSION, formatVersion);
- }
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
deleted file mode 100644
index f36a2e4470..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-import org.apache.iceberg.ParameterizedTestExtension;
-import org.apache.iceberg.Table;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.extension.ExtendWith;
-
-@ExtendWith(ParameterizedTestExtension.class)
-public class TestAlterTableSchema extends ExtensionsTestBase {
-
- @AfterEach
- public void removeTable() {
- sql("DROP TABLE IF EXISTS %s", tableName);
- }
-
- @TestTemplate
- public void testSetIdentifierFields() {
- sql(
- "CREATE TABLE %s (id bigint NOT NULL, "
- + "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg",
- tableName);
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.schema().identifierFieldIds())
- .as("Table should start without identifier")
- .isEmpty();
-
- sql("ALTER TABLE %s SET IDENTIFIER FIELDS id", tableName);
- table.refresh();
- assertThat(table.schema().identifierFieldIds())
- .containsExactly(table.schema().findField("id").fieldId());
-
- sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
- table.refresh();
- assertThat(table.schema().identifierFieldIds())
- .as("Should have new identifier field")
- .containsExactlyInAnyOrder(
- table.schema().findField("id").fieldId(),
- table.schema().findField("location.lon").fieldId());
-
- sql("ALTER TABLE %s SET IDENTIFIER FIELDS location.lon", tableName);
- table.refresh();
- assertThat(table.schema().identifierFieldIds())
- .as("Should have new identifier field")
- .containsExactly(table.schema().findField("location.lon").fieldId());
- }
-
- @TestTemplate
- public void testSetInvalidIdentifierFields() {
- sql("CREATE TABLE %s (id bigint NOT NULL, id2 bigint) USING iceberg", tableName);
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.schema().identifierFieldIds())
- .as("Table should start without identifier")
- .isEmpty();
- assertThatThrownBy(() -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS unknown", tableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageEndingWith("not found in current schema or added columns");
-
- assertThatThrownBy(() -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS id2", tableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageEndingWith("not a required field");
- }
-
- @TestTemplate
- public void testDropIdentifierFields() {
- sql(
- "CREATE TABLE %s (id bigint NOT NULL, "
- + "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg",
- tableName);
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.schema().identifierFieldIds())
- .as("Table should start without identifier")
- .isEmpty();
-
- sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
- table.refresh();
- assertThat(table.schema().identifierFieldIds())
- .as("Should have new identifier fields")
- .containsExactlyInAnyOrder(
- table.schema().findField("id").fieldId(),
- table.schema().findField("location.lon").fieldId());
-
- sql("ALTER TABLE %s DROP IDENTIFIER FIELDS id", tableName);
- table.refresh();
- assertThat(table.schema().identifierFieldIds())
- .as("Should removed identifier field")
- .containsExactly(table.schema().findField("location.lon").fieldId());
-
- sql("ALTER TABLE %s SET IDENTIFIER FIELDS id, location.lon", tableName);
- table.refresh();
- assertThat(table.schema().identifierFieldIds())
- .as("Should have new identifier fields")
- .containsExactlyInAnyOrder(
- table.schema().findField("id").fieldId(),
- table.schema().findField("location.lon").fieldId());
-
- sql("ALTER TABLE %s DROP IDENTIFIER FIELDS id, location.lon", tableName);
- table.refresh();
- assertThat(table.schema().identifierFieldIds()).as("Should have no identifier field").isEmpty();
- }
-
- @TestTemplate
- public void testDropInvalidIdentifierFields() {
- sql(
- "CREATE TABLE %s (id bigint NOT NULL, data string NOT NULL, "
- + "location struct<lon:bigint NOT NULL,lat:bigint NOT NULL> NOT NULL) USING iceberg",
- tableName);
- Table table = validationCatalog.loadTable(tableIdent);
- assertThat(table.schema().identifierFieldIds())
- .as("Table should start without identifier")
- .isEmpty();
- assertThatThrownBy(() -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS unknown", tableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Cannot complete drop identifier fields operation: field unknown not found");
-
- sql("ALTER TABLE %s SET IDENTIFIER FIELDS id", tableName);
- assertThatThrownBy(() -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS data", tableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Cannot complete drop identifier fields operation: data is not an identifier field");
-
- assertThatThrownBy(() -> sql("ALTER TABLE %s DROP IDENTIFIER FIELDS location.lon", tableName))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Cannot complete drop identifier fields operation: location.lon is not an identifier field");
- }
-}
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java
deleted file mode 100644
index 4a3a158dea..0000000000
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAncestorsOfProcedure.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.extensions;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-import java.util.List;
-import org.apache.iceberg.ParameterizedTestExtension;
-import org.apache.iceberg.Table;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.spark.sql.AnalysisException;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.extension.ExtendWith;
-
-@ExtendWith(ParameterizedTestExtension.class)
-public class TestAncestorsOfProcedure extends ExtensionsTestBase {
-
- @AfterEach
- public void removeTables() {
- sql("DROP TABLE IF EXISTS %s", tableName);
- }
-
- @TestTemplate
- public void testAncestorOfUsingEmptyArgs() {
- sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING iceberg", tableName);
- sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);
- sql("INSERT INTO TABLE %s VALUES (2, 'b')", tableName);
-
- Table table = validationCatalog.loadTable(tableIdent);
- Long currentSnapshotId = table.currentSnapshot().snapshotId();
- Long currentTimestamp = table.currentSnapshot().timestampMillis();
- Long preSnapshotId = table.currentSnapshot().parentId();
- Long preTimeStamp = table.snapshot(table.currentSnapshot().parentId()).timestampMillis();
-
... 126377 lines suppressed ...