You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ke...@apache.org on 2025/12/02 16:50:05 UTC
(iceberg) branch main updated: Build: Apply spotless for scala code (#8023)
This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 166a6ebb34 Build: Apply spotless for scala code (#8023)
166a6ebb34 is described below
commit 166a6ebb343cbad85e5cc892c4fe9e880bdc0fb7
Author: Xianyang Liu <xi...@tencent.com>
AuthorDate: Wed Dec 3 00:49:58 2025 +0800
Build: Apply spotless for scala code (#8023)
---
.baseline/scala/.scala212fmt.conf | 32 +++
.baseline/scala/.scala213fmt.conf | 32 +++
baseline.gradle | 15 ++
site/docs/contribute.md | 2 +
.../extensions/IcebergSparkSessionExtensions.scala | 7 +-
.../analysis/AlignRowLevelCommandAssignments.scala | 10 +-
.../AlignedRowLevelIcebergCommandCheck.scala | 1 -
.../analysis/AssignmentAlignmentSupport.scala | 22 ++-
.../analysis/CheckMergeIntoTableConditions.scala | 9 +-
.../spark/sql/catalyst/analysis/CheckViews.scala | 80 +++++---
.../MergeIntoIcebergTableResolutionCheck.scala | 1 -
.../analysis/ProcedureArgumentCoercion.scala | 1 -
.../analysis/ResolveMergeIntoTableReferences.scala | 15 +-
.../sql/catalyst/analysis/ResolveProcedures.scala | 10 +-
.../spark/sql/catalyst/analysis/ResolveViews.scala | 76 +++++---
.../catalyst/analysis/RewriteMergeIntoTable.scala | 42 ++--
.../RewriteMergeIntoTableForRowLineage.scala | 22 ++-
.../analysis/RewriteOperationForRowLineage.scala | 16 +-
.../analysis/RewriteRowLevelIcebergCommand.scala | 6 +-
.../sql/catalyst/analysis/RewriteUpdateTable.scala | 6 +-
.../analysis/RewriteUpdateTableForRowLineage.scala | 7 +-
.../catalyst/analysis/RewriteViewCommands.scala | 69 +++++--
.../spark/sql/catalyst/analysis/ViewUtil.scala | 1 -
.../sql/catalyst/expressions/AssignmentUtils.scala | 24 +--
.../ExtendedReplaceNullWithFalseInPredicate.scala | 21 +-
.../ExtendedSimplifyConditionalsInPredicate.scala | 31 +--
.../RemoveRowLineageOutputFromOriginalTable.scala | 12 +-
.../catalyst/optimizer/ReplaceStaticInvoke.scala | 21 +-
.../IcebergSparkSqlExtensionsParser.scala | 85 ++++----
.../IcebergSqlExtensionsAstBuilder.scala | 215 ++++++++++++---------
.../planning/RewrittenRowLevelCommand.scala | 1 -
.../catalyst/plans/logical/AddPartitionField.scala | 4 +-
.../sql/catalyst/plans/logical/BranchOptions.scala | 8 +-
.../spark/sql/catalyst/plans/logical/Call.scala | 1 -
.../plans/logical/CreateOrReplaceBranch.scala | 4 +-
.../plans/logical/CreateOrReplaceTag.scala | 4 +-
.../sql/catalyst/plans/logical/DropBranch.scala | 1 -
.../plans/logical/DropIdentifierFields.scala | 5 +-
.../plans/logical/DropPartitionField.scala | 1 -
.../spark/sql/catalyst/plans/logical/DropTag.scala | 1 -
.../plans/logical/MergeIntoIcebergTable.scala | 7 +-
.../sql/catalyst/plans/logical/MergeRows.scala | 4 +-
.../catalyst/plans/logical/NoStatsUnaryNode.scala | 1 -
.../plans/logical/ReplaceIcebergData.scala | 11 +-
.../plans/logical/ReplacePartitionField.scala | 6 +-
.../catalyst/plans/logical/RowLevelCommand.scala | 1 -
.../plans/logical/SetIdentifierFields.scala | 5 +-
.../sql/catalyst/plans/logical/TagOptions.scala | 1 -
.../logical/UnresolvedMergeIntoIcebergTable.scala | 8 +-
.../plans/logical/UpdateIcebergTable.scala | 4 +-
.../sql/catalyst/plans/logical/UpdateRows.scala | 4 +-
.../plans/logical/V2WriteCommandLike.scala | 1 -
.../catalyst/plans/logical/WriteIcebergDelta.scala | 22 +--
.../sql/catalyst/plans/logical/statements.scala | 1 -
.../plans/logical/views/CreateIcebergView.scala | 27 +--
.../plans/logical/views/DropIcebergView.scala | 5 +-
.../plans/logical/views/ResolvedV2View.scala | 6 +-
.../plans/logical/views/ShowIcebergViews.scala | 9 +-
.../datasources/v2/AddPartitionFieldExec.scala | 14 +-
.../v2/AlterV2ViewSetPropertiesExec.scala | 13 +-
.../v2/AlterV2ViewUnsetPropertiesExec.scala | 11 +-
.../sql/execution/datasources/v2/CallExec.scala | 7 +-
.../datasources/v2/CreateOrReplaceBranchExec.scala | 25 +--
.../datasources/v2/CreateOrReplaceTagExec.scala | 10 +-
.../datasources/v2/CreateV2ViewExec.scala | 45 ++---
.../datasources/v2/DescribeV2ViewExec.scala | 30 +--
.../execution/datasources/v2/DropBranchExec.scala | 4 +-
.../datasources/v2/DropIdentifierFieldsExec.scala | 25 +--
.../datasources/v2/DropPartitionFieldExec.scala | 27 +--
.../sql/execution/datasources/v2/DropTagExec.scala | 8 +-
.../execution/datasources/v2/DropV2ViewExec.scala | 8 +-
.../v2/ExtendedDataSourceV2Implicits.scala | 1 -
.../v2/ExtendedDataSourceV2Strategy.scala | 89 +++++++--
.../datasources/v2/ExtendedV2Writes.scala | 1 -
.../execution/datasources/v2/MergeRowsExec.scala | 30 +--
.../datasources/v2/RenameV2ViewExec.scala | 9 +-
.../execution/datasources/v2/ReplaceDataExec.scala | 7 +-
.../datasources/v2/ReplacePartitionFieldExec.scala | 30 +--
.../v2/ReplaceRewrittenRowLevelCommand.scala | 1 -
.../v2/RowLevelCommandScanRelationPushDown.scala | 30 +--
.../datasources/v2/SetIdentifierFieldsExec.scala | 13 +-
.../v2/SetWriteDistributionAndOrderingExec.scala | 18 +-
.../datasources/v2/ShowCreateV2ViewExec.scala | 17 +-
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 16 +-
.../execution/datasources/v2/ShowV2ViewsExec.scala | 17 +-
.../execution/datasources/v2/UpdateRowsExec.scala | 6 +-
.../RowLevelCommandDynamicPruning.scala | 30 +--
.../plans/logical/OrderAwareCoalesce.scala | 7 +-
.../logical/SetWriteDistributionAndOrdering.scala | 12 +-
.../plans/logical/SortOrderParserUtil.scala | 5 +-
.../spark/sql/catalyst/utils/PlanUtils.scala | 1 -
.../sql/execution/OrderAwareCoalesceExec.scala | 4 +-
.../datasources/SparkExpressionConverter.scala | 27 ++-
.../apache/spark/sql/stats/ThetaSketchAgg.scala | 13 +-
.../extensions/IcebergSparkSessionExtensions.scala | 7 +-
.../spark/sql/catalyst/analysis/CheckViews.scala | 54 ++++--
.../analysis/ProcedureArgumentCoercion.scala | 1 -
.../sql/catalyst/analysis/ResolveProcedures.scala | 10 +-
.../spark/sql/catalyst/analysis/ResolveViews.scala | 77 ++++----
.../RewriteMergeIntoTableForRowLineage.scala | 21 +-
.../analysis/RewriteOperationForRowLineage.scala | 16 +-
.../analysis/RewriteUpdateTableForRowLineage.scala | 7 +-
.../catalyst/analysis/RewriteViewCommands.scala | 69 +++++--
.../spark/sql/catalyst/analysis/ViewUtil.scala | 1 -
.../RemoveRowLineageOutputFromOriginalTable.scala | 12 +-
.../catalyst/optimizer/ReplaceStaticInvoke.scala | 21 +-
.../IcebergSparkSqlExtensionsParser.scala | 72 +++----
.../IcebergSqlExtensionsAstBuilder.scala | 215 ++++++++++++---------
.../catalyst/plans/logical/AddPartitionField.scala | 4 +-
.../sql/catalyst/plans/logical/BranchOptions.scala | 8 +-
.../spark/sql/catalyst/plans/logical/Call.scala | 1 -
.../plans/logical/CreateOrReplaceBranch.scala | 4 +-
.../plans/logical/CreateOrReplaceTag.scala | 4 +-
.../sql/catalyst/plans/logical/DropBranch.scala | 1 -
.../plans/logical/DropIdentifierFields.scala | 5 +-
.../plans/logical/DropPartitionField.scala | 1 -
.../spark/sql/catalyst/plans/logical/DropTag.scala | 1 -
.../plans/logical/ReplacePartitionField.scala | 6 +-
.../plans/logical/SetIdentifierFields.scala | 5 +-
.../sql/catalyst/plans/logical/TagOptions.scala | 1 -
.../sql/catalyst/plans/logical/statements.scala | 1 -
.../plans/logical/views/CreateIcebergView.scala | 27 +--
.../plans/logical/views/DropIcebergView.scala | 5 +-
.../plans/logical/views/ResolvedV2View.scala | 6 +-
.../plans/logical/views/ShowIcebergViews.scala | 9 +-
.../datasources/v2/AddPartitionFieldExec.scala | 14 +-
.../v2/AlterV2ViewSetPropertiesExec.scala | 13 +-
.../v2/AlterV2ViewUnsetPropertiesExec.scala | 11 +-
.../sql/execution/datasources/v2/CallExec.scala | 7 +-
.../datasources/v2/CreateOrReplaceBranchExec.scala | 25 +--
.../datasources/v2/CreateOrReplaceTagExec.scala | 10 +-
.../datasources/v2/CreateV2ViewExec.scala | 45 ++---
.../datasources/v2/DescribeV2ViewExec.scala | 30 +--
.../execution/datasources/v2/DropBranchExec.scala | 4 +-
.../datasources/v2/DropIdentifierFieldsExec.scala | 25 +--
.../datasources/v2/DropPartitionFieldExec.scala | 27 +--
.../sql/execution/datasources/v2/DropTagExec.scala | 8 +-
.../execution/datasources/v2/DropV2ViewExec.scala | 8 +-
.../v2/ExtendedDataSourceV2Strategy.scala | 50 ++++-
.../datasources/v2/RenameV2ViewExec.scala | 9 +-
.../datasources/v2/ReplacePartitionFieldExec.scala | 30 +--
.../datasources/v2/SetIdentifierFieldsExec.scala | 13 +-
.../v2/SetWriteDistributionAndOrderingExec.scala | 18 +-
.../datasources/v2/ShowCreateV2ViewExec.scala | 17 +-
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 16 +-
.../execution/datasources/v2/ShowV2ViewsExec.scala | 17 +-
.../plans/logical/OrderAwareCoalesce.scala | 7 +-
.../logical/SetWriteDistributionAndOrdering.scala | 12 +-
.../plans/logical/SortOrderParserUtil.scala | 5 +-
.../spark/sql/catalyst/utils/PlanUtils.scala | 1 -
.../sql/execution/OrderAwareCoalesceExec.scala | 4 +-
.../datasources/SparkExpressionConverter.scala | 27 ++-
.../apache/spark/sql/stats/ThetaSketchAgg.scala | 13 +-
.../extensions/IcebergSparkSessionExtensions.scala | 1 -
.../spark/sql/catalyst/analysis/CheckViews.scala | 51 +++--
.../spark/sql/catalyst/analysis/ResolveViews.scala | 77 ++++----
.../catalyst/analysis/RewriteViewCommands.scala | 71 +++++--
.../spark/sql/catalyst/analysis/ViewUtil.scala | 1 -
.../catalyst/optimizer/ReplaceStaticInvoke.scala | 21 +-
.../IcebergSparkSqlExtensionsParser.scala | 73 +++----
.../IcebergSqlExtensionsAstBuilder.scala | 206 +++++++++++---------
.../catalyst/plans/logical/AddPartitionField.scala | 4 +-
.../sql/catalyst/plans/logical/BranchOptions.scala | 8 +-
.../plans/logical/CreateOrReplaceBranch.scala | 4 +-
.../plans/logical/CreateOrReplaceTag.scala | 4 +-
.../sql/catalyst/plans/logical/DropBranch.scala | 1 -
.../plans/logical/DropIdentifierFields.scala | 5 +-
.../plans/logical/DropPartitionField.scala | 1 -
.../spark/sql/catalyst/plans/logical/DropTag.scala | 1 -
.../plans/logical/ReplacePartitionField.scala | 6 +-
.../plans/logical/SetIdentifierFields.scala | 5 +-
.../sql/catalyst/plans/logical/TagOptions.scala | 1 -
.../plans/logical/views/CreateIcebergView.scala | 27 +--
.../plans/logical/views/DropIcebergView.scala | 5 +-
.../plans/logical/views/ResolvedV2View.scala | 6 +-
.../plans/logical/views/ShowIcebergViews.scala | 9 +-
.../datasources/v2/AddPartitionFieldExec.scala | 14 +-
.../v2/AlterV2ViewSetPropertiesExec.scala | 13 +-
.../v2/AlterV2ViewUnsetPropertiesExec.scala | 11 +-
.../datasources/v2/CreateOrReplaceBranchExec.scala | 25 +--
.../datasources/v2/CreateOrReplaceTagExec.scala | 10 +-
.../datasources/v2/CreateV2ViewExec.scala | 45 ++---
.../datasources/v2/DescribeV2ViewExec.scala | 30 +--
.../execution/datasources/v2/DropBranchExec.scala | 4 +-
.../datasources/v2/DropIdentifierFieldsExec.scala | 25 +--
.../datasources/v2/DropPartitionFieldExec.scala | 27 +--
.../sql/execution/datasources/v2/DropTagExec.scala | 8 +-
.../execution/datasources/v2/DropV2ViewExec.scala | 8 +-
.../v2/ExtendedDataSourceV2Strategy.scala | 50 ++++-
.../datasources/v2/RenameV2ViewExec.scala | 9 +-
.../datasources/v2/ReplacePartitionFieldExec.scala | 30 +--
.../datasources/v2/SetIdentifierFieldsExec.scala | 13 +-
.../v2/SetWriteDistributionAndOrderingExec.scala | 18 +-
.../datasources/v2/ShowCreateV2ViewExec.scala | 17 +-
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 16 +-
.../execution/datasources/v2/ShowV2ViewsExec.scala | 17 +-
.../plans/logical/OrderAwareCoalesce.scala | 7 +-
.../logical/SetWriteDistributionAndOrdering.scala | 12 +-
.../plans/logical/SortOrderParserUtil.scala | 5 +-
.../spark/sql/catalyst/utils/PlanUtils.scala | 1 -
.../sql/execution/OrderAwareCoalesceExec.scala | 4 +-
.../datasources/SparkExpressionConverter.scala | 27 ++-
.../apache/spark/sql/stats/ThetaSketchAgg.scala | 13 +-
203 files changed, 2168 insertions(+), 1672 deletions(-)
diff --git a/.baseline/scala/.scala212fmt.conf b/.baseline/scala/.scala212fmt.conf
new file mode 100644
index 0000000000..485a11d6ff
--- /dev/null
+++ b/.baseline/scala/.scala212fmt.conf
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version = 3.9.7
+
+align = none
+align.openParenDefnSite = false
+align.openParenCallSite = false
+align.tokens = []
+importSelectors = "singleLine"
+optIn = {
+ configStyleArguments = false
+}
+danglingParentheses.preset = false
+docstrings.style = Asterisk
+docstrings.wrap = false
+maxColumn = 100
+runner.dialect = scala212
diff --git a/.baseline/scala/.scala213fmt.conf b/.baseline/scala/.scala213fmt.conf
new file mode 100644
index 0000000000..57c44a61c4
--- /dev/null
+++ b/.baseline/scala/.scala213fmt.conf
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version = 3.9.7
+
+align = none
+align.openParenDefnSite = false
+align.openParenCallSite = false
+align.tokens = []
+importSelectors = "singleLine"
+optIn = {
+ configStyleArguments = false
+}
+danglingParentheses.preset = false
+docstrings.style = Asterisk
+docstrings.wrap = false
+maxColumn = 100
+runner.dialect = scala213
diff --git a/baseline.gradle b/baseline.gradle
index 151fa40a1a..54e67b84f8 100644
--- a/baseline.gradle
+++ b/baseline.gradle
@@ -62,6 +62,21 @@ subprojects {
removeUnusedImports()
licenseHeaderFile "$rootDir/.baseline/copyright/copyright-header-java.txt"
}
+
+ // Configure different scalafmt rules for specific Scala version
+ if (project.name.startsWith("iceberg-spark") && project.name.endsWith("2.13")) {
+ scala {
+ target 'src/**/*.scala'
+ scalafmt("3.9.7").configFile("$rootDir/.baseline/scala/.scala213fmt.conf")
+ licenseHeaderFile "$rootDir/.baseline/copyright/copyright-header-java.txt", "package"
+ }
+ } else if (project.name.startsWith("iceberg-spark") && project.name.endsWith("2.12")) {
+ scala {
+ target 'src/**/*.scala'
+ scalafmt("3.9.7").configFile("$rootDir/.baseline/scala/.scala212fmt.conf")
+ licenseHeaderFile "$rootDir/.baseline/copyright/copyright-header-java.txt", "package"
+ }
+ }
}
}
diff --git a/site/docs/contribute.md b/site/docs/contribute.md
index d38cc28c36..54aa076639 100644
--- a/site/docs/contribute.md
+++ b/site/docs/contribute.md
@@ -125,6 +125,8 @@ This project Iceberg also has modules for adding Iceberg support to processing e
Follow the instructions for [Eclipse](https://github.com/google/google-java-format#eclipse) or
[IntelliJ](https://github.com/google/google-java-format#intellij-android-studio-and-other-jetbrains-ides) to install the **google-java-format** plugin (note the required manual actions for IntelliJ).
+Follow the [instructions](https://scalameta.org/scalafmt/docs/installation.html) to install **scalafmt** plugin
+and configure it to point to the configuration file located under the directory `.baseline/scala/`.
## Semantic Versioning
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
index 350ed6988f..c1f77da66f 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.iceberg.spark.extensions
import org.apache.spark.sql.SparkSessionExtensions
@@ -57,8 +56,8 @@ class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
extensions.injectResolutionRule { _ => CheckMergeIntoTableConditions }
extensions.injectResolutionRule { _ => ProcedureArgumentCoercion }
extensions.injectResolutionRule { _ => AlignRowLevelCommandAssignments }
- extensions.injectResolutionRule { _ => RewriteUpdateTableForRowLineage}
- extensions.injectResolutionRule { _ => RewriteMergeIntoTableForRowLineage}
+ extensions.injectResolutionRule { _ => RewriteUpdateTableForRowLineage }
+ extensions.injectResolutionRule { _ => RewriteMergeIntoTableForRowLineage }
extensions.injectResolutionRule { _ => RewriteUpdateTable }
extensions.injectResolutionRule { _ => RewriteMergeIntoTable }
extensions.injectCheckRule { _ => CheckViews }
@@ -69,7 +68,7 @@ class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
extensions.injectOptimizerRule { _ => ExtendedSimplifyConditionalsInPredicate }
extensions.injectOptimizerRule { _ => ExtendedReplaceNullWithFalseInPredicate }
extensions.injectOptimizerRule { _ => ReplaceStaticInvoke }
- extensions.injectOptimizerRule { _ => RemoveRowLineageOutputFromOriginalTable}
+ extensions.injectOptimizerRule { _ => RemoveRowLineageOutputFromOriginalTable }
// pre-CBO rules run only once and the order of the rules is important
// - dynamic filters should be added before replacing commands with rewrite plans
// - scans must be planned before building writes
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala
index 23ba50bdfd..c70ded5ed2 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelCommandAssignments.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -35,8 +34,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
*
* Note that this rule must be run before rewriting row-level commands.
*/
-object AlignRowLevelCommandAssignments
- extends Rule[LogicalPlan] with AssignmentAlignmentSupport {
+object AlignRowLevelCommandAssignments extends Rule[LogicalPlan] with AssignmentAlignmentSupport {
override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
case u: UpdateIcebergTable if u.resolved && !u.aligned =>
@@ -60,7 +58,7 @@ object AlignRowLevelCommandAssignments
if (ref.size > 1) {
throw new AnalysisException(
"Nested fields are not supported inside INSERT clauses of MERGE operations: " +
- s"${ref.mkString("`", "`.`", "`")}")
+ s"${ref.mkString("`", "`.`", "`")}")
}
}
@@ -101,8 +99,8 @@ object AlignRowLevelCommandAssignments
if (assignment.isEmpty) {
throw new AnalysisException(
s"Cannot find column '${targetAttr.name}' of the target table among " +
- s"the INSERT columns: ${assignmentMap.keys.mkString(", ")}. " +
- "INSERT clauses must provide values for all columns of the target table.")
+ s"the INSERT columns: ${assignmentMap.keys.mkString(", ")}. " +
+ "INSERT clauses must provide values for all columns of the target table.")
}
val key = assignment.get.key
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala
index d915e4f109..38021fda29 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignedRowLevelIcebergCommandCheck.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala
index 76aec46a23..0b53075a73 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -69,8 +68,8 @@ trait AssignmentAlignmentSupport extends CastSupport {
val columnUpdates = assignments.map(a => ColumnUpdate(toAssignmentRef(a.key), a.value))
val outputExprs = applyUpdates(table.output, columnUpdates)
- outputExprs.zip(table.output).map {
- case (expr, attr) => handleCharVarcharLimits(Assignment(attr, expr))
+ outputExprs.zip(table.output).map { case (expr, attr) =>
+ handleCharVarcharLimits(Assignment(attr, expr))
}
}
@@ -117,8 +116,7 @@ trait AssignmentAlignmentSupport extends CastSupport {
val colName = (namePrefix :+ col.name).mkString(".")
throw new AnalysisException(
"Updating nested fields is only supported for StructType " +
- s"but $colName is of type $otherType"
- )
+ s"but $colName is of type $otherType")
}
// if there are conflicting updates, throw an exception
@@ -129,7 +127,7 @@ trait AssignmentAlignmentSupport extends CastSupport {
val conflictingCols = updates.map(u => (namePrefix ++ u.ref).mkString("."))
throw new AnalysisException(
"Updates are in conflict for these columns: " +
- conflictingCols.distinct.mkString(", "))
+ conflictingCols.distinct.mkString(", "))
}
}
}
@@ -180,8 +178,13 @@ trait AssignmentAlignmentSupport extends CastSupport {
// e.g. a struct with fields (a, b) is assigned as a struct with fields (a, c) or (b, a)
val errors = new mutable.ArrayBuffer[String]()
val canWrite = DataType.canWrite(
- expr.dataType, tableAttr.dataType, byName = true, resolver, tableAttr.name,
- storeAssignmentPolicy, err => errors += err)
+ expr.dataType,
+ tableAttr.dataType,
+ byName = true,
+ resolver,
+ tableAttr.name,
+ storeAssignmentPolicy,
+ err => errors += err)
if (!canWrite) {
throw new AnalysisException(
@@ -195,7 +198,8 @@ trait AssignmentAlignmentSupport extends CastSupport {
case _ if tableAttr.dataType.sameType(expr.dataType) =>
expr
case StoreAssignmentPolicy.ANSI =>
- val cast = Cast(expr, tableAttr.dataType, Option(conf.sessionLocalTimeZone), ansiEnabled = true)
+ val cast =
+ Cast(expr, tableAttr.dataType, Option(conf.sessionLocalTimeZone), ansiEnabled = true)
cast.setTagValue(Cast.BY_TABLE_INSERTION, ())
TableOutputResolver.checkCastOverflowInTableInsert(cast, colPath.quoted)
case _ =>
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
index 70f6694af6..079a0eee23 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -57,18 +56,20 @@ object CheckMergeIntoTableConditions extends Rule[LogicalPlan] {
if (!cond.deterministic) {
throw new AnalysisException(
s"Non-deterministic functions are not supported in $condName conditions of " +
- s"MERGE operations: ${cond.sql}")
+ s"MERGE operations: ${cond.sql}")
}
if (SubqueryExpression.hasSubquery(cond)) {
throw new AnalysisException(
s"Subqueries are not supported in conditions of MERGE operations. " +
- s"Found a subquery in the $condName condition: ${cond.sql}")
+ s"Found a subquery in the $condName condition: ${cond.sql}")
}
if (cond.find(_.isInstanceOf[AggregateExpression]).isDefined) {
throw new AnalysisException(
- s"Agg functions are not supported in $condName conditions of MERGE operations: " + {cond.sql})
+ s"Agg functions are not supported in $condName conditions of MERGE operations: " + {
+ cond.sql
+ })
}
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
index 44b22e68d0..a7c903dfdb 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -38,44 +37,71 @@ object CheckViews extends (LogicalPlan => Unit) {
override def apply(plan: LogicalPlan): Unit = {
plan foreach {
- case CreateIcebergView(resolvedIdent@ResolvedIdentifier(_: ViewCatalog, _), _, query, columnAliases, _,
- _, _, _, _, replace, _) =>
+ case CreateIcebergView(
+ resolvedIdent @ ResolvedIdentifier(_: ViewCatalog, _),
+ _,
+ query,
+ columnAliases,
+ _,
+ _,
+ _,
+ _,
+ _,
+ replace,
+ _) =>
verifyColumnCount(resolvedIdent, columnAliases, query)
- SchemaUtils.checkColumnNameDuplication(query.schema.fieldNames.toIndexedSeq, SQLConf.get.resolver)
+ SchemaUtils.checkColumnNameDuplication(
+ query.schema.fieldNames.toIndexedSeq,
+ SQLConf.get.resolver)
if (replace) {
- val viewIdent: Seq[String] = resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
+ val viewIdent: Seq[String] =
+ resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
checkCyclicViewReference(viewIdent, query, Seq(viewIdent))
}
case AlterViewAs(ResolvedV2View(_, _), _, _) =>
- throw new AnalysisException("ALTER VIEW <viewName> AS is not supported. Use CREATE OR REPLACE VIEW instead")
+ throw new AnalysisException(
+ "ALTER VIEW <viewName> AS is not supported. Use CREATE OR REPLACE VIEW instead")
case _ => // OK
}
}
- private def verifyColumnCount(ident: ResolvedIdentifier, columns: Seq[String], query: LogicalPlan): Unit = {
+ private def verifyColumnCount(
+ ident: ResolvedIdentifier,
+ columns: Seq[String],
+ query: LogicalPlan): Unit = {
if (columns.nonEmpty) {
if (columns.length > query.output.length) {
- throw new AnalysisException(String.format("Cannot create view %s.%s, the reason is not enough data columns:\n" +
- "View columns: %s\n" +
- "Data columns: %s", ident.catalog.name(), ident.identifier, columns.mkString(", "),
- query.output.map(c => c.name).mkString(", ")))
+ throw new AnalysisException(
+ String.format(
+ "Cannot create view %s.%s, the reason is not enough data columns:\n" +
+ "View columns: %s\n" +
+ "Data columns: %s",
+ ident.catalog.name(),
+ ident.identifier,
+ columns.mkString(", "),
+ query.output.map(c => c.name).mkString(", ")))
} else if (columns.length < query.output.length) {
- throw new AnalysisException(String.format("Cannot create view %s.%s, the reason is too many data columns:\n" +
- "View columns: %s\n" +
- "Data columns: %s", ident.catalog.name(), ident.identifier, columns.mkString(", "),
- query.output.map(c => c.name).mkString(", ")))
+ throw new AnalysisException(
+ String.format(
+ "Cannot create view %s.%s, the reason is too many data columns:\n" +
+ "View columns: %s\n" +
+ "Data columns: %s",
+ ident.catalog.name(),
+ ident.identifier,
+ columns.mkString(", "),
+ query.output.map(c => c.name).mkString(", ")))
}
}
}
private def checkCyclicViewReference(
- viewIdent: Seq[String],
- plan: LogicalPlan,
- cyclePath: Seq[Seq[String]]): Unit = {
+ viewIdent: Seq[String],
+ plan: LogicalPlan,
+ cyclePath: Seq[Seq[String]]): Unit = {
plan match {
- case sub@SubqueryAlias(_, Project(_, _)) =>
+ case sub @ SubqueryAlias(_, Project(_, _)) =>
val currentViewIdent: Seq[String] = sub.identifier.qualifier :+ sub.identifier.name
checkIfRecursiveView(viewIdent, currentViewIdent, cyclePath, sub.children)
case v1View: View =>
@@ -94,15 +120,17 @@ object CheckViews extends (LogicalPlan => Unit) {
}
private def checkIfRecursiveView(
- viewIdent: Seq[String],
- currentViewIdent: Seq[String],
- cyclePath: Seq[Seq[String]],
- children: Seq[LogicalPlan]
- ): Unit = {
+ viewIdent: Seq[String],
+ currentViewIdent: Seq[String],
+ cyclePath: Seq[Seq[String]],
+ children: Seq[LogicalPlan]): Unit = {
val newCyclePath = cyclePath :+ currentViewIdent
if (currentViewIdent == viewIdent) {
- throw new AnalysisException(String.format("Recursive cycle in view detected: %s (cycle: %s)",
- viewIdent.asIdentifier, newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
+ throw new AnalysisException(
+ String.format(
+ "Recursive cycle in view detected: %s (cycle: %s)",
+ viewIdent.asIdentifier,
+ newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
} else {
children.foreach { c =>
checkCyclicViewReference(viewIdent, c, newCyclePath)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala
index b3a9bda280..3cbf27e645 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
index 7f0ca8fadd..aadc332181 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
index bb270391a1..0c64636a00 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -53,7 +52,8 @@ case class ResolveMergeIntoTableReferences(spark: SparkSession) extends Rule[Log
case UpdateAction(cond, assignments) =>
val resolvedCond = cond.map(resolveCond("UPDATE", _, m))
// the update action can access columns from both target and source tables
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = false)
+ val resolvedAssignments =
+ resolveAssignments(assignments, m, resolveValuesWithSourceOnly = false)
UpdateAction(resolvedCond, resolvedAssignments)
case UpdateStarAction(updateCondition) =>
@@ -62,7 +62,8 @@ case class ResolveMergeIntoTableReferences(spark: SparkSession) extends Rule[Log
Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
}
// for UPDATE *, the value must be from the source table
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
+ val resolvedAssignments =
+ resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
UpdateAction(resolvedUpdateCondition, resolvedAssignments)
case _ =>
@@ -74,7 +75,8 @@ case class ResolveMergeIntoTableReferences(spark: SparkSession) extends Rule[Log
// the insert action is used when not matched, so its condition and value can only
// access columns from the source table
val resolvedCond = cond.map(resolveCond("INSERT", _, Project(Nil, m.sourceTable)))
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
+ val resolvedAssignments =
+ resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
InsertAction(resolvedCond, resolvedAssignments)
case InsertStarAction(cond) =>
@@ -84,7 +86,8 @@ case class ResolveMergeIntoTableReferences(spark: SparkSession) extends Rule[Log
val assignments = targetTable.output.map { attr =>
Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
}
- val resolvedAssignments = resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
+ val resolvedAssignments =
+ resolveAssignments(assignments, m, resolveValuesWithSourceOnly = true)
InsertAction(resolvedCond, resolvedAssignments)
case _ =>
@@ -108,7 +111,7 @@ case class ResolveMergeIntoTableReferences(spark: SparkSession) extends Rule[Log
if (unresolvedAttrs.nonEmpty) {
throw new AnalysisException(
s"Cannot resolve ${unresolvedAttrs.map(_.sql).mkString("[", ",", "]")} in $condName condition " +
- s"of MERGE operation given input columns: ${plan.inputSet.toSeq.map(_.sql).mkString("[", ",", "]")}")
+ s"of MERGE operation given input columns: ${plan.inputSet.toSeq.map(_.sql).mkString("[", ",", "]")}")
}
resolvedCond
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
index ee69b5e344..c2d13bfa02 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import java.util.Locale
@@ -61,7 +60,8 @@ case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with
}
if (duplicateParamNames.nonEmpty) {
- throw new AnalysisException(s"Duplicate parameter names: ${duplicateParamNames.mkString("[", ",", "]")}")
+ throw new AnalysisException(
+ s"Duplicate parameter names: ${duplicateParamNames.mkString("[", ",", "]")}")
}
// optional params should be at the end
@@ -89,7 +89,8 @@ case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with
}
if (missingParamNames.nonEmpty) {
- throw new AnalysisException(s"Missing required parameters: ${missingParamNames.mkString("[", ",", "]")}")
+ throw new AnalysisException(
+ s"Missing required parameters: ${missingParamNames.mkString("[", ",", "]")}")
}
val argExprs = new Array[Expression](params.size)
@@ -141,7 +142,8 @@ case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with
}
if (validationErrors.nonEmpty) {
- throw new AnalysisException(s"Could not build name to arg map: ${validationErrors.mkString(", ")}")
+ throw new AnalysisException(
+ s"Could not build name to arg map: ${validationErrors.mkString(", ")}")
}
namedArgs.map(arg => arg.name -> arg).toMap
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
index b77cc03326..1e0e5e8951 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SparkSession
@@ -47,36 +46,52 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case u@UnresolvedRelation(nameParts, _, _)
- if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
+ case u @ UnresolvedRelation(nameParts, _, _)
+ if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
u
- case u@UnresolvedRelation(parts@CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
+ case u @ UnresolvedRelation(parts @ CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
.map(createViewRelation(parts, _))
.getOrElse(u)
- case u@UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
+ case u @ UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
.map(_ => ResolvedV2View(catalog.asViewCatalog, ident))
.getOrElse(u)
- case c@CreateIcebergView(ResolvedIdentifier(_, _), _, query, columnAliases, columnComments, _, _, _, _, _, _)
- if query.resolved && !c.rewritten =>
+ case c @ CreateIcebergView(
+ ResolvedIdentifier(_, _),
+ _,
+ query,
+ columnAliases,
+ columnComments,
+ _,
+ _,
+ _,
+ _,
+ _,
+ _) if query.resolved && !c.rewritten =>
val aliased = aliasColumns(query, columnAliases, columnComments)
- c.copy(query = aliased, queryColumnNames = query.schema.fieldNames.toIndexedSeq, rewritten = true)
+ c.copy(
+ query = aliased,
+ queryColumnNames = query.schema.fieldNames.toIndexedSeq,
+ rewritten = true)
}
private def aliasColumns(
- plan: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]]): LogicalPlan = {
+ plan: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]]): LogicalPlan = {
if (columnAliases.isEmpty || columnAliases.length != plan.output.length) {
plan
} else {
val projectList = plan.output.zipWithIndex.map { case (attr, pos) =>
if (columnComments.apply(pos).isDefined) {
- val meta = new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
+ val meta =
+ new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
Alias(attr, columnAliases.apply(pos))(explicitMetadata = Some(meta))
} else {
Alias(attr, columnAliases.apply(pos))()
@@ -98,17 +113,15 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
// This is more strict because it doesn't allow resolution by field name.
val aliases = view.schema.fields.zipWithIndex.map { case (expected, pos) =>
val attr = GetColumnByOrdinal(pos, expected.dataType)
- Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata = Some(expected.metadata))
+ Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata =
+ Some(expected.metadata))
}.toIndexedSeq
SubqueryAlias(nameParts, Project(aliases, rewritten))
}
private def parseViewText(name: String, viewText: String): LogicalPlan = {
- val origin = Origin(
- objectType = Some("VIEW"),
- objectName = Some(name)
- )
+ val origin = Origin(objectType = Some("VIEW"), objectName = Some(name))
try {
CurrentOrigin.withOrigin(origin) {
@@ -121,8 +134,8 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
private def rewriteIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = {
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = {
// Substitute CTEs and Unresolved Ordinals within the view, then rewrite unresolved functions and relations
qualifyTableIdentifiers(
qualifyFunctionIdentifiers(
@@ -132,15 +145,15 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
private def qualifyFunctionIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
- case u@UnresolvedFunction(Seq(name), _, _, _, _) =>
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
+ case u @ UnresolvedFunction(Seq(name), _, _, _, _) =>
if (!isBuiltinFunction(name)) {
u.copy(nameParts = catalogAndNamespace :+ name)
} else {
u
}
- case u@UnresolvedFunction(parts, _, _, _, _) if !isCatalog(parts.head) =>
+ case u @ UnresolvedFunction(parts, _, _, _, _) if !isCatalog(parts.head) =>
u.copy(nameParts = catalogAndNamespace.head +: parts)
}
@@ -148,17 +161,16 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
* Qualify table identifiers with default catalog and namespace if necessary.
*/
private def qualifyTableIdentifiers(
- child: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan =
+ child: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan =
child transform {
- case u@UnresolvedRelation(Seq(table), _, _) =>
+ case u @ UnresolvedRelation(Seq(table), _, _) =>
u.copy(multipartIdentifier = catalogAndNamespace :+ table)
- case u@UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
+ case u @ UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
u.copy(multipartIdentifier = catalogAndNamespace.head +: parts)
case other =>
- other.transformExpressions {
- case subquery: SubqueryExpression =>
- subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
+ other.transformExpressions { case subquery: SubqueryExpression =>
+ subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
index 4aae44ecf1..7ac9a1c6c8 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -82,7 +81,13 @@ object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with Predicat
private final val ROW_FROM_TARGET_REF = FieldReference(ROW_FROM_TARGET)
override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
+ case m @ MergeIntoIcebergTable(
+ aliasedTable,
+ source,
+ cond,
+ matchedActions,
+ notMatchedActions,
+ None)
if m.resolved && m.aligned && matchedActions.isEmpty && notMatchedActions.size == 1 =>
EliminateSubqueryAliases(aliasedTable) match {
@@ -112,8 +117,13 @@ object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with Predicat
throw new AnalysisException(s"$p is not an Iceberg table")
}
- case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
- if m.resolved && m.aligned && matchedActions.isEmpty =>
+ case m @ MergeIntoIcebergTable(
+ aliasedTable,
+ source,
+ cond,
+ matchedActions,
+ notMatchedActions,
+ None) if m.resolved && m.aligned && matchedActions.isEmpty =>
EliminateSubqueryAliases(aliasedTable) match {
case r: DataSourceV2Relation =>
@@ -145,8 +155,13 @@ object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with Predicat
throw new AnalysisException(s"$p is not an Iceberg table")
}
- case m @ MergeIntoIcebergTable(aliasedTable, source, cond, matchedActions, notMatchedActions, None)
- if m.resolved && m.aligned =>
+ case m @ MergeIntoIcebergTable(
+ aliasedTable,
+ source,
+ cond,
+ matchedActions,
+ notMatchedActions,
+ None) if m.resolved && m.aligned =>
EliminateSubqueryAliases(aliasedTable) match {
case r @ DataSourceV2Relation(tbl: SupportsRowLevelOperations, _, _, _, _) =>
@@ -204,7 +219,8 @@ object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with Predicat
// disable broadcasts for the target table to perform the cardinality check
val joinType = if (notMatchedActions.isEmpty) LeftOuter else FullOuter
val joinHint = JoinHint(leftHint = Some(HintInfo(Some(NO_BROADCAST_HASH))), rightHint = None)
- val joinPlan = Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(cond), joinHint)
+ val joinPlan =
+ Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(cond), joinHint)
val matchedConditions = matchedActions.map(actionCondition)
val matchedOutputs = matchedActions.map(matchedActionOutput(_, metadataAttrs))
@@ -217,7 +233,8 @@ object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with Predicat
val mergeRows = MergeRows(
isSourceRowPresent = IsNotNull(rowFromSourceAttr),
- isTargetRowPresent = if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
+ isTargetRowPresent =
+ if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
matchedConditions = matchedConditions,
matchedOutputs = matchedOutputs,
notMatchedConditions = notMatchedConditions,
@@ -275,7 +292,8 @@ object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with Predicat
// also disable broadcasts for the target table to perform the cardinality check
val joinType = if (notMatchedActions.isEmpty) Inner else RightOuter
val joinHint = JoinHint(leftHint = Some(HintInfo(Some(NO_BROADCAST_HASH))), rightHint = None)
- val joinPlan = Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(joinCond), joinHint)
+ val joinPlan =
+ Join(NoStatsUnaryNode(targetTableProj), sourceTableProj, joinType, Some(joinCond), joinHint)
val metadataReadAttrs = readAttrs.filterNot(relation.outputSet.contains)
@@ -294,11 +312,13 @@ object RewriteMergeIntoTable extends RewriteRowLevelIcebergCommand with Predicat
val rowFromTargetAttr = resolveAttrRef(ROW_FROM_TARGET_REF, joinPlan)
// merged rows must contain values for the operation type and all read attrs
- val mergeRowsOutput = buildMergeRowsOutput(matchedOutputs, notMatchedOutputs, operationTypeAttr +: readAttrs)
+ val mergeRowsOutput =
+ buildMergeRowsOutput(matchedOutputs, notMatchedOutputs, operationTypeAttr +: readAttrs)
val mergeRows = MergeRows(
isSourceRowPresent = IsNotNull(rowFromSourceAttr),
- isTargetRowPresent = if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
+ isTargetRowPresent =
+ if (notMatchedActions.isEmpty) TrueLiteral else IsNotNull(rowFromTargetAttr),
matchedConditions = matchedConditions,
matchedOutputs = matchedOutputs,
notMatchedConditions = notMatchedConditions,
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala
index cebb360a29..1dbc171f5d 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.catalyst.expressions.Literal
@@ -32,9 +31,9 @@ object RewriteMergeIntoTableForRowLineage extends RewriteOperationForRowLineage
override def apply(plan: LogicalPlan): LogicalPlan = {
plan.resolveOperators {
case m @ MergeIntoIcebergTable(_, _, _, matchedActions, _, _)
- if m.resolved && m.aligned &&
- matchedActions.nonEmpty &&
- shouldUpdatePlan(m.targetTable) =>
+ if m.resolved && m.aligned &&
+ matchedActions.nonEmpty &&
+ shouldUpdatePlan(m.targetTable) =>
updateMergeIntoForRowLineage(m)
}
}
@@ -48,17 +47,22 @@ object RewriteMergeIntoTableForRowLineage extends RewriteOperationForRowLineage
val matchedAssignmentsForLineage = matchedActions.map {
case UpdateAction(cond, assignments) =>
- UpdateAction(cond, assignments ++ Seq(Assignment(rowId, rowId),
- Assignment(lastUpdatedSequenceNumber, Literal(null))))
+ UpdateAction(
+ cond,
+ assignments ++ Seq(
+ Assignment(rowId, rowId),
+ Assignment(lastUpdatedSequenceNumber, Literal(null))))
case deleteAction => deleteAction
}
val notMatchedActionsForLineage = notMatchedActions.map {
case InsertAction(cond, assignments) =>
- InsertAction(cond, assignments ++ Seq(
- Assignment(rowId, Literal(null)),
- Assignment(lastUpdatedSequenceNumber, Literal(null))))
+ InsertAction(
+ cond,
+ assignments ++ Seq(
+ Assignment(rowId, Literal(null)),
+ Assignment(lastUpdatedSequenceNumber, Literal(null))))
}
val tableWithLineage = r.copy(output = r.output ++ Seq(rowId, lastUpdatedSequenceNumber))
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala
index 611e62b75e..538e042647 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.iceberg.MetadataColumns
@@ -32,7 +31,8 @@ import org.apache.spark.sql.types.MetadataBuilder
trait RewriteOperationForRowLineage extends RewriteRowLevelIcebergCommand {
protected val ROW_ID_ATTRIBUTE_NAME = MetadataColumns.ROW_ID.name()
- protected val LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name()
+ protected val LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME =
+ MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name()
// The plan should only be updated if the underlying Iceberg table supports row lineage AND
// lineage attributes are not already on the output of operation which indicates the rule already ran
@@ -52,16 +52,17 @@ trait RewriteOperationForRowLineage extends RewriteRowLevelIcebergCommand {
}
protected def findRowLineageAttributes(
- expressions: Seq[Expression]
- ): Option[(AttributeReference, AttributeReference)] = {
+ expressions: Seq[Expression]): Option[(AttributeReference, AttributeReference)] = {
val rowIdAttr = expressions.collectFirst {
case attr: AttributeReference
- if isMetadataColumn(attr) && attr.name == ROW_ID_ATTRIBUTE_NAME => attr
+ if isMetadataColumn(attr) && attr.name == ROW_ID_ATTRIBUTE_NAME =>
+ attr
}
val lastUpdatedAttr = expressions.collectFirst {
case attr: AttributeReference
- if isMetadataColumn(attr) && attr.name == LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME => attr
+ if isMetadataColumn(attr) && attr.name == LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME =>
+ attr
}
// Treat row lineage columns as data columns by removing the metadata attribute
@@ -78,7 +79,8 @@ trait RewriteOperationForRowLineage extends RewriteRowLevelIcebergCommand {
attr.withMetadata(
new MetadataBuilder()
.withMetadata(attr.metadata)
- .remove(METADATA_COL_ATTR_KEY).build())
+ .remove(METADATA_COL_ATTR_KEY)
+ .build())
}
private def isMetadataColumn(attributeReference: AttributeReference): Boolean = {
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelIcebergCommand.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelIcebergCommand.scala
index 0b1871038f..f40fc16e8c 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelIcebergCommand.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelIcebergCommand.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -169,9 +168,8 @@ trait RewriteRowLevelIcebergCommand extends RewriteRowLevelCommand {
operation match {
case supportsDelta: SupportsDelta =>
- val rowIdAttrs = V2ExpressionUtils.resolveRefs[AttributeReference](
- supportsDelta.rowId.toSeq,
- relation)
+ val rowIdAttrs =
+ V2ExpressionUtils.resolveRefs[AttributeReference](supportsDelta.rowId.toSeq, relation)
val nullableRowIdAttrs = rowIdAttrs.filter(_.nullable)
if (nullableRowIdAttrs.nonEmpty) {
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala
index bbe5532185..664f4d8da6 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTable.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -152,9 +151,8 @@ object RewriteUpdateTable extends RewriteRowLevelIcebergCommand {
// build a plan for updated records that match the cond
val matchedRowsPlan = Filter(cond, readRelation)
- val updatedRowsPlan = updateRows(
- matchedRowsPlan, assignments, readAttrs,
- rowAttrs, rowIdAttrs, metadataReadAttrs)
+ val updatedRowsPlan =
+ updateRows(matchedRowsPlan, assignments, readAttrs, rowAttrs, rowIdAttrs, metadataReadAttrs)
// build a plan to write the row delta to the table
val writeRelation = relation.copy(table = operationTable)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala
index 00513d89fc..34f4de6bb3 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.catalyst.expressions.Literal
@@ -30,17 +29,17 @@ object RewriteUpdateTableForRowLineage extends RewriteOperationForRowLineage {
override def apply(plan: LogicalPlan): LogicalPlan = {
plan resolveOperators {
- case updateTable@UpdateIcebergTable(_, _, _, _) if shouldUpdatePlan(updateTable.table) =>
+ case updateTable @ UpdateIcebergTable(_, _, _, _) if shouldUpdatePlan(updateTable.table) =>
updatePlanWithRowLineage(updateTable)
}
}
-
private def updatePlanWithRowLineage(updateTable: UpdateIcebergTable): LogicalPlan = {
EliminateSubqueryAliases(updateTable.table) match {
case r @ DataSourceV2Relation(_: SupportsRowLevelOperations, _, _, _, _) =>
val lineageAttributes = findRowLineageAttributes(r.metadataOutput).get
- val (rowId, lastUpdatedSequence) = (removeMetadataColumnAttribute(lineageAttributes._1),
+ val (rowId, lastUpdatedSequence) = (
+ removeMetadataColumnAttribute(lineageAttributes._1),
removeMetadataColumnAttribute(lineageAttributes._2))
val lineageAssignments = updateTable.assignments ++
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
index c7f09b2f5a..e5cad5ee10 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -52,11 +51,19 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case DropView(ResolvedIdent(resolved), ifExists) =>
DropIcebergView(resolved, ifExists)
- case CreateView(ResolvedIdent(resolved), userSpecifiedColumns, comment, properties,
- Some(queryText), query, allowExisting, replace) =>
+ case CreateView(
+ ResolvedIdent(resolved),
+ userSpecifiedColumns,
+ comment,
+ properties,
+ Some(queryText),
+ query,
+ allowExisting,
+ replace) =>
val q = CTESubstitution.apply(query)
verifyTemporaryObjectsDontExist(resolved, q)
- CreateIcebergView(child = resolved,
+ CreateIcebergView(
+ child = resolved,
queryText = queryText,
query = q,
columnAliases = userSpecifiedColumns.map(_._1),
@@ -68,20 +75,25 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case view @ ShowViews(UnresolvedNamespace(Seq()), pattern, output) =>
if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) {
- ShowIcebergViews(ResolvedNamespace(catalogManager.currentCatalog, catalogManager.currentNamespace.toIndexedSeq),
- pattern, output)
+ ShowIcebergViews(
+ ResolvedNamespace(
+ catalogManager.currentCatalog,
+ catalogManager.currentNamespace.toIndexedSeq),
+ pattern,
+ output)
} else {
view
}
case ShowViews(UnresolvedNamespace(CatalogAndNamespace(catalog, ns)), pattern, output)
- if ViewUtil.isViewCatalog(catalog) =>
+ if ViewUtil.isViewCatalog(catalog) =>
ShowIcebergViews(ResolvedNamespace(catalog, ns), pattern, output)
// needs to be done here instead of in ResolveViews, so that a V2 view can be resolved before the Analyzer
// tries to resolve it, which would result in an error, saying that V2 views aren't supported
- case u@UnresolvedView(ResolvedView(resolved), _, _, _) =>
- ViewUtil.loadView(resolved.catalog, resolved.identifier)
+ case u @ UnresolvedView(ResolvedView(resolved), _, _, _) =>
+ ViewUtil
+ .loadView(resolved.catalog, resolved.identifier)
.map(_ => ResolvedV2View(resolved.catalog.asViewCatalog, resolved.identifier))
.getOrElse(u)
}
@@ -102,7 +114,8 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case UnresolvedIdentifier(nameParts, true) if isTempView(nameParts) =>
None
- case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _) if ViewUtil.isViewCatalog(catalog) =>
+ case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _)
+ if ViewUtil.isViewCatalog(catalog) =>
Some(ResolvedIdentifier(catalog, ident))
case _ =>
@@ -114,11 +127,14 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
* Permanent views are not allowed to reference temp objects
*/
private def verifyTemporaryObjectsDontExist(
- identifier: ResolvedIdentifier,
- child: LogicalPlan): Unit = {
+ identifier: ResolvedIdentifier,
+ child: LogicalPlan): Unit = {
val tempViews = collectTemporaryViews(child)
if (tempViews.nonEmpty) {
- throw invalidRefToTempObject(identifier, tempViews.map(v => v.quoted).mkString("[", ", ", "]"), "view")
+ throw invalidRefToTempObject(
+ identifier,
+ tempViews.map(v => v.quoted).mkString("[", ", ", "]"),
+ "view")
}
val tempFunctions = collectTemporaryFunctions(child)
@@ -127,9 +143,17 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
}
}
- private def invalidRefToTempObject(ident: ResolvedIdentifier, tempObjectNames: String, tempObjectType: String) = {
- new AnalysisException(String.format("Cannot create view %s.%s that references temporary %s: %s",
- ident.catalog.name(), ident.identifier, tempObjectType, tempObjectNames))
+ private def invalidRefToTempObject(
+ ident: ResolvedIdentifier,
+ tempObjectNames: String,
+ tempObjectType: String) = {
+ new AnalysisException(
+ String.format(
+ "Cannot create view %s.%s that references temporary %s: %s",
+ ident.catalog.name(),
+ ident.identifier,
+ tempObjectType,
+ tempObjectNames))
}
/**
@@ -141,10 +165,11 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case unresolved: UnresolvedRelation if isTempView(unresolved.multipartIdentifier) =>
Seq(unresolved.multipartIdentifier)
case view: View if view.isTempView => Seq(view.desc.identifier.nameParts)
- case plan => plan.expressions.flatMap(_.flatMap {
- case e: SubqueryExpression => collectTempViews(e.plan)
- case _ => Seq.empty
- })
+ case plan =>
+ plan.expressions.flatMap(_.flatMap {
+ case e: SubqueryExpression => collectTempViews(e.plan)
+ case _ => Seq.empty
+ })
}.distinct
}
@@ -157,7 +182,9 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
None
case CatalogAndIdentifier(catalog, ident) if ViewUtil.isViewCatalog(catalog) =>
- ViewUtil.loadView(catalog, ident).flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
+ ViewUtil
+ .loadView(catalog, ident)
+ .flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
case _ =>
None
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
index d46f10b7f5..a7188837c5 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.connector.catalog.CatalogPlugin
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala
index ce3818922c..89c0630556 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AssignmentUtils.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.AnalysisException
@@ -40,12 +39,13 @@ object AssignmentUtils extends SQLConfHelper {
sameSize && table.output.zip(assignments).forall { case (attr, assignment) =>
val key = assignment.key
val value = assignment.value
- val refsEqual = toAssignmentRef(attr).zip(toAssignmentRef(key))
- .forall{ case (attrRef, keyRef) => conf.resolver(attrRef, keyRef)}
+ val refsEqual = toAssignmentRef(attr)
+ .zip(toAssignmentRef(key))
+ .forall { case (attrRef, keyRef) => conf.resolver(attrRef, keyRef) }
refsEqual &&
- DataType.equalsIgnoreCompatibleNullability(value.dataType, attr.dataType) &&
- (attr.nullable || !value.nullable)
+ DataType.equalsIgnoreCompatibleNullability(value.dataType, attr.dataType) &&
+ (attr.nullable || !value.nullable)
}
}
@@ -66,16 +66,16 @@ object AssignmentUtils extends SQLConfHelper {
val key = assignment.key
val value = assignment.value
- val rawKeyType = key.transform {
- case attr: AttributeReference =>
- CharVarcharUtils.getRawType(attr.metadata)
- .map(attr.withDataType)
- .getOrElse(attr)
+ val rawKeyType = key.transform { case attr: AttributeReference =>
+ CharVarcharUtils
+ .getRawType(attr.metadata)
+ .map(attr.withDataType)
+ .getOrElse(attr)
}.dataType
if (CharVarcharUtils.hasCharVarchar(rawKeyType)) {
- val newKey = key.transform {
- case attr: AttributeReference => CharVarcharUtils.cleanAttrMetadata(attr)
+ val newKey = key.transform { case attr: AttributeReference =>
+ CharVarcharUtils.cleanAttrMetadata(attr)
}
val newValue = CharVarcharUtils.stringLengthCheck(value, rawKeyType)
Assignment(newKey, newValue)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
index 803eb5d257..ad11f927f1 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.expressions.And
@@ -50,18 +49,18 @@ import org.apache.spark.util.Utils
*/
object ExtendedReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
- _.containsAnyPattern(NULL_LITERAL, TRUE_OR_FALSE_LITERAL, INSET)) {
+ override def apply(plan: LogicalPlan): LogicalPlan =
+ plan.transformWithPruning(_.containsAnyPattern(NULL_LITERAL, TRUE_OR_FALSE_LITERAL, INSET)) {
- case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
- u.copy(condition = Some(replaceNullWithFalse(cond)))
+ case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
+ u.copy(condition = Some(replaceNullWithFalse(cond)))
- case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
- m.copy(
- mergeCondition = replaceNullWithFalse(mergeCond),
- matchedActions = replaceNullWithFalse(matchedActions),
- notMatchedActions = replaceNullWithFalse(notMatchedActions))
- }
+ case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
+ m.copy(
+ mergeCondition = replaceNullWithFalse(mergeCond),
+ matchedActions = replaceNullWithFalse(matchedActions),
+ notMatchedActions = replaceNullWithFalse(notMatchedActions))
+ }
/**
* Recursively traverse the Boolean-type expression to replace
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
index ac6df514d3..ba5f6c8dfb 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.expressions.And
@@ -40,18 +39,18 @@ import org.apache.spark.sql.types.BooleanType
*/
object ExtendedSimplifyConditionalsInPredicate extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
- _.containsAnyPattern(CASE_WHEN, IF)) {
+ override def apply(plan: LogicalPlan): LogicalPlan =
+ plan.transformWithPruning(_.containsAnyPattern(CASE_WHEN, IF)) {
- case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
- u.copy(condition = Some(simplifyConditional(cond)))
+ case u @ UpdateIcebergTable(_, _, Some(cond), _) =>
+ u.copy(condition = Some(simplifyConditional(cond)))
- case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
- m.copy(
- mergeCondition = simplifyConditional(mergeCond),
- matchedActions = simplifyConditional(matchedActions),
- notMatchedActions = simplifyConditional(notMatchedActions))
- }
+ case m @ MergeIntoIcebergTable(_, _, mergeCond, matchedActions, notMatchedActions, _) =>
+ m.copy(
+ mergeCondition = simplifyConditional(mergeCond),
+ matchedActions = simplifyConditional(matchedActions),
+ notMatchedActions = simplifyConditional(notMatchedActions))
+ }
private def simplifyConditional(e: Expression): Expression = e match {
case And(left, right) => And(simplifyConditional(left), simplifyConditional(right))
@@ -61,8 +60,9 @@ object ExtendedSimplifyConditionalsInPredicate extends Rule[LogicalPlan] {
case If(cond, FalseLiteral, falseValue) =>
And(Not(Coalesce(Seq(cond, FalseLiteral))), falseValue)
case If(cond, TrueLiteral, falseValue) => Or(cond, falseValue)
- case CaseWhen(Seq((cond, trueValue)),
- Some(FalseLiteral) | Some(Literal(null, BooleanType)) | None) =>
+ case CaseWhen(
+ Seq((cond, trueValue)),
+ Some(FalseLiteral) | Some(Literal(null, BooleanType)) | None) =>
And(cond, trueValue)
case CaseWhen(Seq((cond, trueValue)), Some(TrueLiteral)) =>
Or(Not(Coalesce(Seq(cond, FalseLiteral))), trueValue)
@@ -72,9 +72,10 @@ object ExtendedSimplifyConditionalsInPredicate extends Rule[LogicalPlan] {
Or(cond, elseValue)
case e if e.dataType == BooleanType => e
case e =>
- assert(e.dataType != BooleanType,
+ assert(
+ e.dataType != BooleanType,
"Expected a Boolean type expression in ExtendedSimplifyConditionalsInPredicate, " +
- s"but got the type `${e.dataType.catalogString}` in `${e.sql}`.")
+ s"but got the type `${e.dataType.catalogString}` in `${e.sql}`.")
e
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala
index a3e9ad372b..b6af8ae799 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.optimizer
import org.apache.iceberg.MetadataColumns
@@ -35,20 +34,19 @@ import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
object RemoveRowLineageOutputFromOriginalTable extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = {
plan.transform {
- case writeDelta@WriteIcebergDelta(_, _, originalTable, _, _) =>
+ case writeDelta @ WriteIcebergDelta(_, _, originalTable, _, _) =>
writeDelta.copy(originalTable = removeRowLineageOutput(originalTable))
- case replaceData@ReplaceIcebergData(_, _, originalTable, _) =>
+ case replaceData @ ReplaceIcebergData(_, _, originalTable, _) =>
replaceData.copy(originalTable = removeRowLineageOutput(originalTable))
}
}
private def removeRowLineageOutput(table: NamedRelation): DataSourceV2Relation = {
table match {
- case dsv2Relation@DataSourceV2Relation(_, _, _, _, _) =>
- dsv2Relation.copy(output = dsv2Relation.output.filterNot(
- attr => attr.name == MetadataColumns.ROW_ID.name() ||
+ case dsv2Relation @ DataSourceV2Relation(_, _, _, _, _) =>
+ dsv2Relation.copy(output = dsv2Relation.output.filterNot(attr =>
+ attr.name == MetadataColumns.ROW_ID.name() ||
attr.name == MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name()))
}
}
}
-
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
index 5ad26bcaae..a2a68d864b 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
@@ -47,13 +47,13 @@ import org.apache.spark.sql.types.StructType
object ReplaceStaticInvoke extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan =
- plan.transformWithPruning (_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
+ plan.transformWithPruning(_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
case join @ Join(_, _, _, Some(cond), _) =>
replaceStaticInvoke(join, cond, newCond => join.copy(condition = Some(newCond)))
case filter @ Filter(cond, _) =>
replaceStaticInvoke(filter, cond, newCond => filter.copy(condition = newCond))
- }
+ }
private def replaceStaticInvoke[T <: LogicalPlan](
node: T,
@@ -86,16 +86,17 @@ object ReplaceStaticInvoke extends Rule[LogicalPlan] {
return invoke
}
- val inputType = StructType(invoke.arguments.zipWithIndex.map {
- case (exp, pos) => StructField(s"_$pos", exp.dataType, exp.nullable)
+ val inputType = StructType(invoke.arguments.zipWithIndex.map { case (exp, pos) =>
+ StructField(s"_$pos", exp.dataType, exp.nullable)
})
- val bound = try {
- unbound.bind(inputType)
- } catch {
- case _: Exception =>
- return invoke
- }
+ val bound =
+ try {
+ unbound.bind(inputType)
+ } catch {
+ case _: Exception =>
+ return invoke
+ }
if (bound.inputTypes().length != invoke.arguments.length) {
return invoke
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
index e3f42c12fa..1fb0641c00 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.parser.extensions
import java.util.Locale
@@ -59,7 +58,9 @@ import org.apache.spark.sql.types.StructType
import scala.jdk.CollectionConverters._
import scala.util.Try
-class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserInterface with ExtendedParser {
+class IcebergSparkSqlExtensionsParser(delegate: ParserInterface)
+ extends ParserInterface
+ with ExtendedParser {
import IcebergSparkSqlExtensionsParser._
@@ -128,7 +129,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
override def parsePlan(sqlText: String): LogicalPlan = {
val sqlTextAfterSubstitution = substitutor.substitute(sqlText)
if (isIcebergCommand(sqlTextAfterSubstitution)) {
- parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }.asInstanceOf[LogicalPlan]
+ parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }
+ .asInstanceOf[LogicalPlan]
} else {
val parsedPlan = RewriteViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText))
parsedPlan match {
@@ -144,7 +146,13 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
case UpdateTable(UnresolvedIcebergTable(aliasedTable), assignments, condition) =>
UpdateIcebergTable(aliasedTable, assignments, condition)
- case MergeIntoTable(UnresolvedIcebergTable(aliasedTable), source, cond, matchedActions, notMatchedActions, Nil) =>
+ case MergeIntoTable(
+ UnresolvedIcebergTable(aliasedTable),
+ source,
+ cond,
+ matchedActions,
+ notMatchedActions,
+ Nil) =>
// cannot construct MergeIntoIcebergTable right away as MERGE operations require special resolution
// that's why the condition and actions must be hidden from the regular resolution rules in Spark
// see ResolveMergeIntoTableReferences for details
@@ -168,7 +176,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
}
private def isIcebergTable(multipartIdent: Seq[String]): Boolean = {
- val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(SparkSession.active, multipartIdent.asJava)
+ val catalogAndIdentifier =
+ Spark3Util.catalogAndIdentifier(SparkSession.active, multipartIdent.asJava)
catalogAndIdentifier.catalog match {
case tableCatalog: TableCatalog =>
Try(tableCatalog.loadTable(catalogAndIdentifier.identifier))
@@ -187,7 +196,9 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
}
private def isIcebergCommand(sqlText: String): Boolean = {
- val normalized = sqlText.toLowerCase(Locale.ROOT).trim()
+ val normalized = sqlText
+ .toLowerCase(Locale.ROOT)
+ .trim()
// Strip simple SQL comments that terminate a line, e.g. comments starting with `--` .
.replaceAll("--.*?\\n", " ")
// Strip newlines.
@@ -198,37 +209,37 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
// Strip backtick then `system`.`ancestors_of` changes to system.ancestors_of
.replaceAll("`", "")
.trim()
- isIcebergProcedure(normalized) || (
- normalized.startsWith("alter table") && (
- normalized.contains("add partition field") ||
- normalized.contains("drop partition field") ||
- normalized.contains("replace partition field") ||
- normalized.contains("write ordered by") ||
- normalized.contains("write locally ordered by") ||
- normalized.contains("write distributed by") ||
- normalized.contains("write unordered") ||
- normalized.contains("set identifier fields") ||
- normalized.contains("drop identifier fields") ||
- isSnapshotRefDdl(normalized)))
+ isIcebergProcedure(normalized) || (normalized
+ .startsWith("alter table") && (normalized.contains("add partition field") ||
+ normalized.contains("drop partition field") ||
+ normalized.contains("replace partition field") ||
+ normalized.contains("write ordered by") ||
+ normalized.contains("write locally ordered by") ||
+ normalized.contains("write distributed by") ||
+ normalized.contains("write unordered") ||
+ normalized.contains("set identifier fields") ||
+ normalized.contains("drop identifier fields") ||
+ isSnapshotRefDdl(normalized)))
}
// All builtin Iceberg procedures are under the 'system' namespace
private def isIcebergProcedure(normalized: String): Boolean = {
normalized.startsWith("call") &&
- SparkProcedures.names().asScala.map("system." + _).exists(normalized.contains)
+ SparkProcedures.names().asScala.map("system." + _).exists(normalized.contains)
}
private def isSnapshotRefDdl(normalized: String): Boolean = {
normalized.contains("create branch") ||
- normalized.contains("replace branch") ||
- normalized.contains("create tag") ||
- normalized.contains("replace tag") ||
- normalized.contains("drop branch") ||
- normalized.contains("drop tag")
+ normalized.contains("replace branch") ||
+ normalized.contains("create tag") ||
+ normalized.contains("replace tag") ||
+ normalized.contains("drop branch") ||
+ normalized.contains("drop tag")
}
protected def parse[T](command: String)(toResult: IcebergSqlExtensionsParser => T): T = {
- val lexer = new IcebergSqlExtensionsLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
+ val lexer = new IcebergSqlExtensionsLexer(
+ new UpperCaseCharStream(CharStreams.fromString(command)))
lexer.removeErrorListeners()
lexer.addErrorListener(IcebergParseErrorListener)
@@ -246,8 +257,7 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
parser.setErrorHandler(new BailErrorStrategy)
parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
toResult(parser)
- }
- catch {
+ } catch {
case _: ParseCancellationException =>
// if we fail, parse with LL mode with DefaultErrorStrategy
tokenStream.seek(0) // rewind input stream
@@ -258,8 +268,7 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
parser.getInterpreter.setPredictionMode(PredictionMode.LL)
toResult(parser)
}
- }
- catch {
+ } catch {
case e: IcebergParseException if e.command.isDefined =>
throw e
case e: IcebergParseException =>
@@ -277,7 +286,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
object IcebergSparkSqlExtensionsParser {
private val substitutorCtor: DynConstructors.Ctor[VariableSubstitution] =
- DynConstructors.builder()
+ DynConstructors
+ .builder()
.impl(classOf[VariableSubstitution])
.impl(classOf[VariableSubstitution], classOf[SQLConf])
.build()
@@ -323,9 +333,7 @@ case object IcebergSqlExtensionsPostProcessor extends IcebergSqlExtensionsBaseLi
replaceTokenByIdentifier(ctx, 0)(identity)
}
- private def replaceTokenByIdentifier(
- ctx: ParserRuleContext,
- stripMargins: Int)(
+ private def replaceTokenByIdentifier(ctx: ParserRuleContext, stripMargins: Int)(
f: CommonToken => CommonToken = identity): Unit = {
val parent = ctx.getParent
parent.removeLastChild()
@@ -372,10 +380,12 @@ class IcebergParseException(
val command: Option[String],
message: String,
val start: Origin,
- val stop: Origin) extends AnalysisException(message, start.line, start.startPosition) {
+ val stop: Origin)
+ extends AnalysisException(message, start.line, start.startPosition) {
def this(message: String, ctx: ParserRuleContext) = {
- this(Option(IcebergParserUtils.command(ctx)),
+ this(
+ Option(IcebergParserUtils.command(ctx)),
message,
IcebergParserUtils.position(ctx.getStart),
IcebergParserUtils.position(ctx.getStop))
@@ -385,8 +395,7 @@ class IcebergParseException(
val builder = new StringBuilder
builder ++= "\n" ++= message
start match {
- case Origin(
- Some(l), Some(p), Some(_), Some(_), Some(_), Some(_), Some(_)) =>
+ case Origin(Some(l), Some(p), Some(_), Some(_), Some(_), Some(_), Some(_)) =>
builder ++= s"(line $l, pos $p)\n"
command.foreach { cmd =>
val (above, below) = cmd.split("\n").splitAt(l)
@@ -406,4 +415,4 @@ class IcebergParseException(
def withCommand(cmd: String): IcebergParseException = {
new IcebergParseException(Option(cmd), message, start, stop)
}
-}
\ No newline at end of file
+}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
index 6b1cc41da0..f87733e6b7 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.parser.extensions
import java.util.Locale
@@ -63,9 +62,11 @@ import org.apache.spark.sql.connector.expressions.LiteralValue
import org.apache.spark.sql.connector.expressions.Transform
import scala.jdk.CollectionConverters._
-class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
+class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface)
+ extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
- private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] = list.asScala
+ private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] =
+ list.asScala
private def toSeq[T](list: java.util.List[T]): Seq[T] = toBuffer(list).toSeq
/**
@@ -80,52 +81,59 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an ADD PARTITION FIELD logical command.
*/
- override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField = withOrigin(ctx) {
- AddPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform),
- Option(ctx.name).map(_.getText))
- }
+ override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField =
+ withOrigin(ctx) {
+ AddPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform),
+ Option(ctx.name).map(_.getText))
+ }
/**
* Create a DROP PARTITION FIELD logical command.
*/
- override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField = withOrigin(ctx) {
- DropPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform))
- }
+ override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField =
+ withOrigin(ctx) {
+ DropPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform))
+ }
/**
* Create a CREATE OR REPLACE BRANCH logical command.
*/
- override def visitCreateOrReplaceBranch(ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
+ override def visitCreateOrReplaceBranch(
+ ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
val createOrReplaceBranchClause = ctx.createReplaceBranchClause()
val branchName = createOrReplaceBranchClause.identifier()
val branchOptionsContext = Option(createOrReplaceBranchClause.branchOptions())
- val snapshotId = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotId()))
+ val snapshotId = branchOptionsContext
+ .flatMap(branchOptions => Option(branchOptions.snapshotId()))
.map(_.getText.toLong)
- val snapshotRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
- val minSnapshotsToKeep = snapshotRetention.flatMap(retention => Option(retention.minSnapshotsToKeep()))
+ val snapshotRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
+ val minSnapshotsToKeep = snapshotRetention
+ .flatMap(retention => Option(retention.minSnapshotsToKeep()))
.map(minSnapshots => minSnapshots.number().getText.toLong)
val maxSnapshotAgeMs = snapshotRetention
.flatMap(retention => Option(retention.maxSnapshotAge()))
- .map(retention => TimeUnit.valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
- .toMillis(retention.number().getText.toLong))
- val branchRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
+ .map(retention =>
+ TimeUnit
+ .valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retention.number().getText.toLong))
+ val branchRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
val branchRefAgeMs = branchRetention.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
val create = createOrReplaceBranchClause.CREATE() != null
val replace = ctx.createReplaceBranchClause().REPLACE() != null
val ifNotExists = createOrReplaceBranchClause.EXISTS() != null
- val branchOptions = BranchOptions(
- snapshotId,
- minSnapshotsToKeep,
- maxSnapshotAgeMs,
- branchRefAgeMs
- )
+ val branchOptions =
+ BranchOptions(snapshotId, minSnapshotsToKeep, maxSnapshotAgeMs, branchRefAgeMs)
CreateOrReplaceBranch(
typedVisit[Seq[String]](ctx.multipartIdentifier),
@@ -139,52 +147,61 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an CREATE OR REPLACE TAG logical command.
*/
- override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag = withOrigin(ctx) {
- val createTagClause = ctx.createReplaceTagClause()
-
- val tagName = createTagClause.identifier().getText
-
- val tagOptionsContext = Option(createTagClause.tagOptions())
- val snapshotId = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.snapshotId()))
- .map(_.getText.toLong)
- val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
- val tagRefAgeMs = tagRetain.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
- val tagOptions = TagOptions(
- snapshotId,
- tagRefAgeMs
- )
-
- val create = createTagClause.CREATE() != null
- val replace = createTagClause.REPLACE() != null
- val ifNotExists = createTagClause.EXISTS() != null
-
- CreateOrReplaceTag(typedVisit[Seq[String]](ctx.multipartIdentifier),
- tagName,
- tagOptions,
- create,
- replace,
- ifNotExists)
- }
+ override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag =
+ withOrigin(ctx) {
+ val createTagClause = ctx.createReplaceTagClause()
+
+ val tagName = createTagClause.identifier().getText
+
+ val tagOptionsContext = Option(createTagClause.tagOptions())
+ val snapshotId = tagOptionsContext
+ .flatMap(tagOptions => Option(tagOptions.snapshotId()))
+ .map(_.getText.toLong)
+ val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
+ val tagRefAgeMs = tagRetain.map(retain =>
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
+ val tagOptions = TagOptions(snapshotId, tagRefAgeMs)
+
+ val create = createTagClause.CREATE() != null
+ val replace = createTagClause.REPLACE() != null
+ val ifNotExists = createTagClause.EXISTS() != null
+
+ CreateOrReplaceTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ tagName,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists)
+ }
/**
* Create an DROP BRANCH logical command.
*/
override def visitDropBranch(ctx: DropBranchContext): DropBranch = withOrigin(ctx) {
- DropBranch(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
+ DropBranch(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
}
/**
* Create an DROP TAG logical command.
*/
override def visitDropTag(ctx: DropTagContext): DropTag = withOrigin(ctx) {
- DropTag(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
+ DropTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
}
/**
* Create an REPLACE PARTITION FIELD logical command.
*/
- override def visitReplacePartitionField(ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
+ override def visitReplacePartitionField(
+ ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
ReplacePartitionField(
typedVisit[Seq[String]](ctx.multipartIdentifier),
typedVisit[Transform](ctx.transform(0)),
@@ -195,20 +212,22 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an SET IDENTIFIER FIELDS logical command.
*/
- override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields = withOrigin(ctx) {
- SetIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
+ override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields =
+ withOrigin(ctx) {
+ SetIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
/**
* Create an DROP IDENTIFIER FIELDS logical command.
*/
- override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields = withOrigin(ctx) {
- DropIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
+ override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields =
+ withOrigin(ctx) {
+ DropIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
/**
* Create a [[SetWriteDistributionAndOrdering]] for changing the write distribution and ordering.
@@ -266,12 +285,15 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
*/
override def visitOrderField(ctx: OrderFieldContext): (Term, SortDirection, NullOrder) = {
val term = Spark3Util.toIcebergTerm(typedVisit[Transform](ctx.transform))
- val direction = Option(ctx.ASC).map(_ => SortDirection.ASC)
- .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
- .getOrElse(SortDirection.ASC)
- val nullOrder = Option(ctx.FIRST).map(_ => NullOrder.NULLS_FIRST)
- .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
- .getOrElse(if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
+ val direction = Option(ctx.ASC)
+ .map(_ => SortDirection.ASC)
+ .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
+ .getOrElse(SortDirection.ASC)
+ val nullOrder = Option(ctx.FIRST)
+ .map(_ => NullOrder.NULLS_FIRST)
+ .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
+ .getOrElse(
+ if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
(term, direction, nullOrder)
}
@@ -293,35 +315,40 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create a transform argument from a column reference or a constant.
*/
- override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression = withOrigin(ctx) {
- val reference = Option(ctx.multipartIdentifier())
+ override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression =
+ withOrigin(ctx) {
+ val reference = Option(ctx.multipartIdentifier())
.map(typedVisit[Seq[String]])
.map(FieldReference(_))
- val literal = Option(ctx.constant)
+ val literal = Option(ctx.constant)
.map(visitConstant)
.map(lit => LiteralValue(lit.value, lit.dataType))
- reference.orElse(literal)
+ reference
+ .orElse(literal)
.getOrElse(throw new IcebergParseException(s"Invalid transform argument", ctx))
- }
+ }
/**
* Return a multi-part identifier as Seq[String].
*/
- override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) {
- toSeq(ctx.parts).map(_.getText)
- }
+ override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] =
+ withOrigin(ctx) {
+ toSeq(ctx.parts).map(_.getText)
+ }
- override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] = withOrigin(ctx) {
- toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
- }
+ override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] =
+ withOrigin(ctx) {
+ toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
+ }
/**
* Create a positional argument in a stored procedure call.
*/
- override def visitPositionalArgument(ctx: PositionalArgumentContext): CallArgument = withOrigin(ctx) {
- val expr = typedVisit[Expression](ctx.expression)
- PositionalArgument(expr)
- }
+ override def visitPositionalArgument(ctx: PositionalArgumentContext): CallArgument =
+ withOrigin(ctx) {
+ val expr = typedVisit[Expression](ctx.expression)
+ PositionalArgument(expr)
+ }
/**
* Create a named argument in a stored procedure call.
@@ -350,10 +377,12 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
}
private def reconstructSqlString(ctx: ParserRuleContext): String = {
- toBuffer(ctx.children).map {
- case c: ParserRuleContext => reconstructSqlString(c)
- case t: TerminalNode => t.getText
- }.mkString(" ")
+ toBuffer(ctx.children)
+ .map {
+ case c: ParserRuleContext => reconstructSqlString(c)
+ case t: TerminalNode => t.getText
+ }
+ .mkString(" ")
}
private def typedVisit[T](ctx: ParseTree): T = {
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
index b6b39fd77e..623251d122 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.planning
import org.apache.spark.sql.AnalysisException
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
index e8b1b29411..0a830dbd4f 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
@@ -16,13 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.expressions.Transform
-case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String]) extends LeafCommand {
+case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String])
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
index 4d7e0a086b..15b9083002 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
@@ -16,8 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
-case class BranchOptions (snapshotId: Option[Long], numSnapshots: Option[Long],
- snapshotRetain: Option[Long], snapshotRefRetain: Option[Long])
+case class BranchOptions(
+ snapshotId: Option[Long],
+ numSnapshots: Option[Long],
+ snapshotRetain: Option[Long],
+ snapshotRefRetain: Option[Long])
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
index 9616dae5a8..e80715e57b 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
index b7981a3c7a..6900f6e8cc 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -27,7 +26,8 @@ case class CreateOrReplaceBranch(
branchOptions: BranchOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
+ ifNotExists: Boolean)
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
index 6e7db84a90..957c68e7a5 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -27,7 +26,8 @@ case class CreateOrReplaceTag(
tagOptions: TagOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
+ ifNotExists: Boolean)
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
index bee0b0fae6..ed4f1f512b 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
index 29dd686a0f..1a91806280 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
@@ -16,14 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
-case class DropIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
+case class DropIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
index fb14513241..ec952c8c71 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
index 7e4b38e74d..da69ca0383 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
index 8f84851dcd..9e2734ab1f 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
@@ -28,7 +27,8 @@ case class MergeIntoIcebergTable(
mergeCondition: Expression,
matchedActions: Seq[MergeAction],
notMatchedActions: Seq[MergeAction],
- rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
+ rewritePlan: Option[LogicalPlan] = None)
+ extends RowLevelCommand {
lazy val aligned: Boolean = {
val matchedActionsAligned = matchedActions.forall {
@@ -71,7 +71,8 @@ case class MergeIntoIcebergTable(
case Seq(newTarget, newSource, newRewritePlan) =>
copy(targetTable = newTarget, sourceTable = newSource, rewritePlan = Some(newRewritePlan))
case _ =>
- throw new IllegalArgumentException("MergeIntoIcebergTable expects either two or three children")
+ throw new IllegalArgumentException(
+ "MergeIntoIcebergTable expects either two or three children")
}
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
index 57f8bf3583..d808737d10 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -35,7 +34,8 @@ case class MergeRows(
performCardinalityCheck: Boolean,
emitNotMatchedTargetRows: Boolean,
output: Seq[Attribute],
- child: LogicalPlan) extends UnaryNode {
+ child: LogicalPlan)
+ extends UnaryNode {
require(targetOutput.nonEmpty || !emitNotMatchedTargetRows)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala
index c21df71f06..9368734667 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceIcebergData.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceIcebergData.scala
index 2b741bef12..6648193229 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceIcebergData.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceIcebergData.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.analysis.NamedRelation
@@ -33,7 +32,8 @@ case class ReplaceIcebergData(
table: NamedRelation,
query: LogicalPlan,
originalTable: NamedRelation,
- write: Option[Write] = None) extends V2WriteCommandLike {
+ write: Option[Write] = None)
+ extends V2WriteCommandLike {
override lazy val references: AttributeSet = query.outputSet
override lazy val stringArgs: Iterator[Any] = Iterator(table, query, write)
@@ -45,7 +45,8 @@ case class ReplaceIcebergData(
}
override def outputResolved: Boolean = {
- assert(table.resolved && query.resolved,
+ assert(
+ table.resolved && query.resolved,
"`outputResolved` can only be called when `table` and `query` are both resolved.")
// take into account only incoming data columns and ignore metadata columns in the query
@@ -58,8 +59,8 @@ case class ReplaceIcebergData(
val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
// names and types must match, nullability must be compatible
inAttr.name == outAttr.name &&
- DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outType) &&
- (outAttr.nullable || !inAttr.nullable)
+ DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outType) &&
+ (outAttr.nullable || !inAttr.nullable)
})
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
index 8c660c6f37..c2525369e7 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -26,13 +25,14 @@ case class ReplacePartitionField(
table: Seq[String],
transformFrom: Transform,
transformTo: Transform,
- name: Option[String]) extends LeafCommand {
+ name: Option[String])
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
override def simpleString(maxFields: Int): String = {
s"ReplacePartitionField ${table.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala
index 837ee963bc..ffddce9f3e 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Expression
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
index 1e5622e9c3..8cd2c0ddad 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
@@ -16,14 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
-case class SetIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
+case class SetIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
index 85e3b95f4a..6afe1478d7 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
case class TagOptions(snapshotId: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala
index 895aa733ff..1ecb727007 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Expression
@@ -27,14 +26,17 @@ import org.apache.spark.sql.catalyst.expressions.Expression
case class UnresolvedMergeIntoIcebergTable(
targetTable: LogicalPlan,
sourceTable: LogicalPlan,
- context: MergeIntoContext) extends BinaryCommand {
+ context: MergeIntoContext)
+ extends BinaryCommand {
def duplicateResolved: Boolean = targetTable.outputSet.intersect(sourceTable.outputSet).isEmpty
override def left: LogicalPlan = targetTable
override def right: LogicalPlan = sourceTable
- override protected def withNewChildrenInternal(newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan = {
+ override protected def withNewChildrenInternal(
+ newLeft: LogicalPlan,
+ newRight: LogicalPlan): LogicalPlan = {
copy(targetTable = newLeft, sourceTable = newRight)
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala
index 790eb9380e..58aef899d6 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateIcebergTable.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.AssignmentUtils
@@ -26,7 +25,8 @@ case class UpdateIcebergTable(
table: LogicalPlan,
assignments: Seq[Assignment],
condition: Option[Expression],
- rewritePlan: Option[LogicalPlan] = None) extends RowLevelCommand {
+ rewritePlan: Option[LogicalPlan] = None)
+ extends RowLevelCommand {
lazy val aligned: Boolean = AssignmentUtils.aligned(table, assignments)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateRows.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateRows.scala
index 0446f30404..268150e6b0 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateRows.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateRows.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -28,7 +27,8 @@ case class UpdateRows(
deleteOutput: Seq[Expression],
insertOutput: Seq[Expression],
output: Seq[Attribute],
- child: LogicalPlan) extends UnaryNode {
+ child: LogicalPlan)
+ extends UnaryNode {
override lazy val producedAttributes: AttributeSet = {
AttributeSet(output.filterNot(attr => inputSet.contains(attr)))
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
index 9192d74b7c..e34d1e2daa 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.analysis.NamedRelation
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteIcebergDelta.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteIcebergDelta.scala
index 8495856fb6..66ece84f88 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteIcebergDelta.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/WriteIcebergDelta.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.AnalysisException
@@ -44,7 +43,8 @@ case class WriteIcebergDelta(
query: LogicalPlan,
originalTable: NamedRelation,
projections: WriteDeltaProjections,
- write: Option[DeltaWrite] = None) extends V2WriteCommandLike {
+ write: Option[DeltaWrite] = None)
+ extends V2WriteCommandLike {
override protected lazy val stringArgs: Iterator[Any] = Iterator(table, query, write)
@@ -71,18 +71,17 @@ case class WriteIcebergDelta(
table.skipSchemaResolution || (projections.rowProjection match {
case Some(projection) =>
table.output.size == projection.schema.size &&
- projection.schema.zip(table.output).forall { case (field, outAttr) =>
- isCompatible(field, outAttr)
- }
+ projection.schema.zip(table.output).forall { case (field, outAttr) =>
+ isCompatible(field, outAttr)
+ }
case None =>
true
})
}
private def rowIdAttrsResolved: Boolean = {
- val rowIdAttrs = V2ExpressionUtils.resolveRefs[AttributeReference](
- operation.rowId.toSeq,
- originalTable)
+ val rowIdAttrs =
+ V2ExpressionUtils.resolveRefs[AttributeReference](operation.rowId.toSeq, originalTable)
projections.rowIdProjection.schema.forall { field =>
rowIdAttrs.exists(rowIdAttr => isCompatible(field, rowIdAttr))
@@ -109,12 +108,13 @@ case class WriteIcebergDelta(
val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
// names and types must match, nullability must be compatible
projectionField.name == outAttr.name &&
- DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
- (outAttr.nullable || !projectionField.nullable)
+ DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
+ (outAttr.nullable || !projectionField.nullable)
}
override def outputResolved: Boolean = {
- assert(table.resolved && query.resolved,
+ assert(
+ table.resolved && query.resolved,
"`outputResolved` can only be called when `table` and `query` are both resolved.")
operationResolved && rowAttrsResolved && rowIdAttrsResolved && metadataAttrsResolved
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index be15f32bc1..9986b30c45 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Expression
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
index 9366d5efe1..c5dadae701 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
@@ -16,29 +16,30 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.plans.logical.BinaryCommand
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
case class CreateIcebergView(
- child: LogicalPlan,
- queryText: String,
- query: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String] = Seq.empty,
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean,
- rewritten: Boolean = false) extends BinaryCommand {
+ child: LogicalPlan,
+ queryText: String,
+ query: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String] = Seq.empty,
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean,
+ rewritten: Boolean = false)
+ extends BinaryCommand {
override def left: LogicalPlan = child
override def right: LogicalPlan = query
override protected def withNewChildrenInternal(
- newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+ newLeft: LogicalPlan,
+ newRight: LogicalPlan): LogicalPlan =
copy(child = newLeft, query = newRight)
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
index 275dba6fbf..092b6b33fb 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
@@ -16,15 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
-case class DropIcebergView(
- child: LogicalPlan,
- ifExists: Boolean) extends UnaryCommand {
+case class DropIcebergView(child: LogicalPlan, ifExists: Boolean) extends UnaryCommand {
override protected def withNewChildInternal(newChild: LogicalPlan): DropIcebergView =
copy(child = newChild)
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
index b9c05ff006..4d384e8577 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.analysis.LeafNodeWithoutStats
@@ -24,8 +23,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-case class ResolvedV2View(
- catalog: ViewCatalog,
- identifier: Identifier) extends LeafNodeWithoutStats {
+case class ResolvedV2View(catalog: ViewCatalog, identifier: Identifier)
+ extends LeafNodeWithoutStats {
override def output: Seq[Attribute] = Nil
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
index b09c27acdc..cbfe23d94c 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
@@ -16,8 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -26,9 +24,10 @@ import org.apache.spark.sql.catalyst.plans.logical.ShowViews
import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
case class ShowIcebergViews(
- namespace: LogicalPlan,
- pattern: Option[String],
- override val output: Seq[Attribute] = ShowViews.getOutputAttrs) extends UnaryCommand {
+ namespace: LogicalPlan,
+ pattern: Option[String],
+ override val output: Seq[Attribute] = ShowViews.getOutputAttrs)
+ extends UnaryCommand {
override def child: LogicalPlan = namespace
override protected def withNewChildInternal(newChild: LogicalPlan): ShowIcebergViews =
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
index 55f327f7e4..e28dcfb194 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -31,7 +30,8 @@ case class AddPartitionFieldExec(
catalog: TableCatalog,
ident: Identifier,
transform: Transform,
- name: Option[String]) extends LeafV2CommandExec {
+ name: Option[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -39,12 +39,14 @@ case class AddPartitionFieldExec(
override protected def run(): Seq[InternalRow] = {
catalog.loadTable(ident) match {
case iceberg: SparkTable =>
- iceberg.table.updateSpec()
- .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
+ .commit()
case table =>
- throw new UnsupportedOperationException(s"Cannot add partition field to non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot add partition field to non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
index b103d1ee2c..d6630e51ff 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,17 +24,17 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.connector.catalog.ViewChange
-
case class AlterV2ViewSetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- properties: Map[String, String]) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ properties: Map[String, String])
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
override protected def run(): Seq[InternalRow] = {
- val changes = properties.map {
- case (property, value) => ViewChange.setProperty(property, value)
+ val changes = properties.map { case (property, value) =>
+ ViewChange.setProperty(property, value)
}.toSeq
catalog.alterView(ident, changes: _*)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
index 91abd95f57..7ebf0b9f70 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.AnalysisException
@@ -26,12 +25,12 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.connector.catalog.ViewChange
-
case class AlterV2ViewUnsetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- propertyKeys: Seq[String],
- ifExists: Boolean) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ propertyKeys: Seq[String],
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
index f66962a8c4..a122775025 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,10 +24,8 @@ import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.connector.iceberg.catalog.Procedure
import scala.collection.compat.immutable.ArraySeq
-case class CallExec(
- output: Seq[Attribute],
- procedure: Procedure,
- input: InternalRow) extends LeafV2CommandExec {
+case class CallExec(output: Seq[Attribute], procedure: Procedure, input: InternalRow)
+ extends LeafV2CommandExec {
override protected def run(): Seq[InternalRow] = {
ArraySeq.unsafeWrapArray(procedure.call(input))
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
index 2be406e7f3..baf985f53a 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -28,13 +27,14 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
case class CreateOrReplaceBranchExec(
- catalog: TableCatalog,
- ident: Identifier,
- branch: String,
- branchOptions: BranchOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
+ catalog: TableCatalog,
+ ident: Identifier,
+ branch: String,
+ branchOptions: BranchOptions,
+ create: Boolean,
+ replace: Boolean,
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -62,8 +62,10 @@ case class CreateOrReplaceBranchExec(
if (create && replace && !refExists) {
safeCreateBranch()
} else if (replace) {
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete replace branch operation on %s, main has no snapshot", ident)
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete replace branch operation on %s, main has no snapshot",
+ ident)
manageSnapshots.replaceBranch(branch, snapshotId)
} else {
if (refExists && ifNotExists) {
@@ -88,7 +90,8 @@ case class CreateOrReplaceBranchExec(
manageSnapshots.commit()
case table =>
- throw new UnsupportedOperationException(s"Cannot create or replace branch on non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot create or replace branch on non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
index 372cd75486..e486892614 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -33,7 +32,8 @@ case class CreateOrReplaceTagExec(
tagOptions: TagOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -47,8 +47,10 @@ case class CreateOrReplaceTagExec(
.map(java.lang.Long.valueOf)
.orNull
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete create or replace tag operation on %s, main has no snapshot", ident)
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete create or replace tag operation on %s, main has no snapshot",
+ ident)
val manageSnapshot = iceberg.table.manageSnapshots()
val refExists = null != iceberg.table().refs().get(tag)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
index eb4cc7de48..c9d4047658 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.SupportsReplaceView
@@ -29,31 +28,33 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.types.StructType
import scala.jdk.CollectionConverters._
-
case class CreateV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- queryText: String,
- viewSchema: StructType,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String],
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ queryText: String,
+ viewSchema: StructType,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String],
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
override protected def run(): Seq[InternalRow] = {
val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name
- val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
+ val currentCatalog =
+ if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
val currentNamespace = session.sessionState.catalogManager.currentNamespace
val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION
val newProperties = properties ++
comment.map(ViewCatalog.PROP_COMMENT -> _) ++
- Map(ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
+ Map(
+ ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
ViewCatalog.PROP_ENGINE_VERSION -> engineVersion)
if (replace) {
@@ -87,10 +88,10 @@ case class CreateV2ViewExec(
}
private def replaceView(
- supportsReplaceView: SupportsReplaceView,
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
+ supportsReplaceView: SupportsReplaceView,
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
supportsReplaceView.replaceView(
ident,
queryText,
@@ -104,9 +105,9 @@ case class CreateV2ViewExec(
}
private def createView(
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
catalog.createView(
ident,
queryText,
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
index a9c7312d4a..f25ac8e3c3 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -27,10 +26,9 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
-case class DescribeV2ViewExec(
- output: Seq[Attribute],
- view: View,
- isExtended: Boolean) extends V2CommandExec with LeafExecNode {
+case class DescribeV2ViewExec(output: Seq[Attribute], view: View, isExtended: Boolean)
+ extends V2CommandExec
+ with LeafExecNode {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -44,23 +42,22 @@ case class DescribeV2ViewExec(
private def describeSchema: Seq[InternalRow] =
view.schema().map { column =>
- toCatalystRow(
- column.name,
- column.dataType.simpleString,
- column.getComment().getOrElse(""))
+ toCatalystRow(column.name, column.dataType.simpleString, column.getComment().getOrElse(""))
}
private def emptyRow: InternalRow = toCatalystRow("", "", "")
private def describeExtended: Seq[InternalRow] = {
val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
- val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
+ val properties: Map[String, String] =
+ view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2)
- val viewProperties = properties.toSeq.sortBy(_._1).map {
- case (key, value) =>
+ val viewProperties = properties.toSeq
+ .sortBy(_._1)
+ .map { case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
- }.mkString("[", ", ", "]")
-
+ }
+ .mkString("[", ", ", "]")
// omitting view text here because it is shown as
// part of SHOW CREATE TABLE and can result in weird formatting in the DESCRIBE output
@@ -69,7 +66,10 @@ case class DescribeV2ViewExec(
toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
toCatalystRow("View Query Output Columns", outputColumns, "") ::
toCatalystRow("View Properties", viewProperties, "") ::
- toCatalystRow("Created By", view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""), "") ::
+ toCatalystRow(
+ "Created By",
+ view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""),
+ "") ::
Nil
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
index ff8f182009..e7d9c7b70d 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -29,7 +28,8 @@ case class DropBranchExec(
catalog: TableCatalog,
ident: Identifier,
branch: String,
- ifExists: Boolean) extends LeafV2CommandExec {
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
index dee778b474..87b18594d5 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -27,10 +26,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
-case class DropIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
+case class DropIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -42,18 +39,24 @@ case class DropIdentifierFieldsExec(
val identifierFieldNames = Sets.newHashSet(schema.identifierFieldNames)
for (name <- fields) {
- Preconditions.checkArgument(schema.findField(name) != null,
- "Cannot complete drop identifier fields operation: field %s not found", name)
- Preconditions.checkArgument(identifierFieldNames.contains(name),
- "Cannot complete drop identifier fields operation: %s is not an identifier field", name)
+ Preconditions.checkArgument(
+ schema.findField(name) != null,
+ "Cannot complete drop identifier fields operation: field %s not found",
+ name)
+ Preconditions.checkArgument(
+ identifierFieldNames.contains(name),
+ "Cannot complete drop identifier fields operation: %s is not an identifier field",
+ name)
identifierFieldNames.remove(name)
}
- iceberg.table.updateSchema()
+ iceberg.table
+ .updateSchema()
.setIdentifierFields(identifierFieldNames)
.commit();
case table =>
- throw new UnsupportedOperationException(s"Cannot drop identifier fields in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot drop identifier fields in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
index 9a153f0c00..db43263e0e 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -29,10 +28,8 @@ import org.apache.spark.sql.connector.expressions.FieldReference
import org.apache.spark.sql.connector.expressions.IdentityTransform
import org.apache.spark.sql.connector.expressions.Transform
-case class DropPartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transform: Transform) extends LeafV2CommandExec {
+case class DropPartitionFieldExec(catalog: TableCatalog, ident: Identifier, transform: Transform)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -42,20 +39,24 @@ case class DropPartitionFieldExec(
case iceberg: SparkTable =>
val schema = iceberg.table.schema
transform match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
// the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .commit()
case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transform))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transform))
+ .commit()
}
case table =>
- throw new UnsupportedOperationException(s"Cannot drop partition field in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot drop partition field in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
index 8df88765a9..79b4a15255 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -25,11 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
-case class DropTagExec(
- catalog: TableCatalog,
- ident: Identifier,
- tag: String,
- ifExists: Boolean) extends LeafV2CommandExec {
+case class DropTagExec(catalog: TableCatalog, ident: Identifier, tag: String, ifExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
index c35af1486f..6dd1188b78 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,11 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-case class DropV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- ifExists: Boolean) extends LeafV2CommandExec {
+case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala
index 85bda0b08d..c960505a51 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.AnalysisException
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
index a87b215ce6..afdddb7fab 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -77,11 +76,28 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
AddPartitionFieldExec(catalog, ident, transform, name) :: Nil
case CreateOrReplaceBranch(
- IcebergCatalogAndIdentifier(catalog, ident), branch, branchOptions, create, replace, ifNotExists) =>
- CreateOrReplaceBranchExec(catalog, ident, branch, branchOptions, create, replace, ifNotExists) :: Nil
+ IcebergCatalogAndIdentifier(catalog, ident),
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) =>
+ CreateOrReplaceBranchExec(
+ catalog,
+ ident,
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) :: Nil
case CreateOrReplaceTag(
- IcebergCatalogAndIdentifier(catalog, ident), tag, tagOptions, create, replace, ifNotExists) =>
+ IcebergCatalogAndIdentifier(catalog, ident),
+ tag,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists) =>
CreateOrReplaceTagExec(catalog, ident, tag, tagOptions, create, replace, ifNotExists) :: Nil
case DropBranch(IcebergCatalogAndIdentifier(catalog, ident), branch, ifExists) =>
@@ -93,7 +109,11 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
case DropPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform) =>
DropPartitionFieldExec(catalog, ident, transform) :: Nil
- case ReplacePartitionField(IcebergCatalogAndIdentifier(catalog, ident), transformFrom, transformTo, name) =>
+ case ReplacePartitionField(
+ IcebergCatalogAndIdentifier(catalog, ident),
+ transformFrom,
+ transformTo,
+ name) =>
ReplacePartitionFieldExec(catalog, ident, transformFrom, transformTo, name) :: Nil
case SetIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
@@ -103,24 +123,49 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
DropIdentifierFieldsExec(catalog, ident, fields) :: Nil
case SetWriteDistributionAndOrdering(
- IcebergCatalogAndIdentifier(catalog, ident), distributionMode, ordering) =>
+ IcebergCatalogAndIdentifier(catalog, ident),
+ distributionMode,
+ ordering) =>
SetWriteDistributionAndOrderingExec(catalog, ident, distributionMode, ordering) :: Nil
case ReplaceIcebergData(_: DataSourceV2Relation, query, r: DataSourceV2Relation, Some(write)) =>
// refresh the cache using the original relation
ReplaceDataExec(planLater(query), refreshCache(r), write) :: Nil
- case WriteIcebergDelta(_: DataSourceV2Relation, query, r: DataSourceV2Relation, projs, Some(write)) =>
+ case WriteIcebergDelta(
+ _: DataSourceV2Relation,
+ query,
+ r: DataSourceV2Relation,
+ projs,
+ Some(write)) =>
// refresh the cache using the original relation
WriteDeltaExec(planLater(query), refreshCache(r), projs, write) :: Nil
- case MergeRows(isSourceRowPresent, isTargetRowPresent, matchedConditions, matchedOutputs, notMatchedConditions,
- notMatchedOutputs, targetOutput, performCardinalityCheck, emitNotMatchedTargetRows,
- output, child) =>
-
- MergeRowsExec(isSourceRowPresent, isTargetRowPresent, matchedConditions, matchedOutputs, notMatchedConditions,
- notMatchedOutputs, targetOutput, performCardinalityCheck, emitNotMatchedTargetRows,
- output, planLater(child)) :: Nil
+ case MergeRows(
+ isSourceRowPresent,
+ isTargetRowPresent,
+ matchedConditions,
+ matchedOutputs,
+ notMatchedConditions,
+ notMatchedOutputs,
+ targetOutput,
+ performCardinalityCheck,
+ emitNotMatchedTargetRows,
+ output,
+ child) =>
+
+ MergeRowsExec(
+ isSourceRowPresent,
+ isTargetRowPresent,
+ matchedConditions,
+ matchedOutputs,
+ notMatchedConditions,
+ notMatchedOutputs,
+ targetOutput,
+ performCardinalityCheck,
+ emitNotMatchedTargetRows,
+ output,
+ planLater(child)) :: Nil
case UpdateRows(deleteOutput, insertOutput, output, child) =>
UpdateRowsExec(deleteOutput, insertOutput, output, planLater(child)) :: Nil
@@ -131,7 +176,7 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
case OrderAwareCoalesce(numPartitions, coalescer, child) =>
OrderAwareCoalesceExec(numPartitions, coalescer, planLater(child)) :: Nil
- case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView@true) =>
+ case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView @ true) =>
val newIdent = Spark3Util.catalogAndIdentifier(spark, newName.toList.asJava)
if (oldCatalog.name != newIdent.catalog().name()) {
throw new AnalysisException(
@@ -142,8 +187,18 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
case DropIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), ifExists) =>
DropV2ViewExec(viewCatalog, ident, ifExists) :: Nil
- case CreateIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), queryText, query,
- columnAliases, columnComments, queryColumnNames, comment, properties, allowExisting, replace, _) =>
+ case CreateIcebergView(
+ ResolvedIdentifier(viewCatalog: ViewCatalog, ident),
+ queryText,
+ query,
+ columnAliases,
+ columnComments,
+ queryColumnNames,
+ comment,
+ properties,
+ allowExisting,
+ replace,
+ _) =>
CreateV2ViewExec(
catalog = viewCatalog,
ident = ident,
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala
index 0d13f6a523..98b93b476b 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedV2Writes.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import java.util.Optional
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala
index 474a417d33..a183b1903c 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.SparkException
@@ -45,7 +44,8 @@ case class MergeRowsExec(
performCardinalityCheck: Boolean,
emitNotMatchedTargetRows: Boolean,
output: Seq[Attribute],
- child: SparkPlan) extends UnaryExecNode {
+ child: SparkPlan)
+ extends UnaryExecNode {
private final val ROW_ID = "__row_id"
@@ -115,12 +115,20 @@ case class MergeRowsExec(
val mergeIterator = if (matchedActions.exists(_.isInstanceOf[Split])) {
new SplittingMergeRowIterator(
- rowIterator, cardinalityCheck, isTargetRowPresentPred,
- matchedActions, notMatchedActions)
+ rowIterator,
+ cardinalityCheck,
+ isTargetRowPresentPred,
+ matchedActions,
+ notMatchedActions)
} else {
new MergeRowIterator(
- rowIterator, cardinalityCheck, isTargetRowPresentPred, isSourceRowPresentPred,
- projectTargetCols, matchedActions.asInstanceOf[Seq[Project]], notMatchedActions)
+ rowIterator,
+ cardinalityCheck,
+ isTargetRowPresentPred,
+ isSourceRowPresentPred,
+ projectTargetCols,
+ matchedActions.asInstanceOf[Seq[Project]],
+ notMatchedActions)
}
// null indicates a record must be discarded
@@ -152,7 +160,7 @@ case class MergeRowsExec(
private val targetTableProj: Projection,
private val matchedActions: Seq[Project],
private val notMatchedActions: Seq[Project])
- extends Iterator[InternalRow] {
+ extends Iterator[InternalRow] {
override def hasNext: Boolean = rowIterator.hasNext
@@ -206,7 +214,7 @@ case class MergeRowsExec(
private val isTargetRowPresentPred: BasePredicate,
private val matchedActions: Seq[Action],
private val notMatchedActions: Seq[Project])
- extends Iterator[InternalRow] {
+ extends Iterator[InternalRow] {
var cachedExtraRow: InternalRow = _
@@ -266,9 +274,9 @@ case class MergeRowsExec(
protected def fail(): Unit = {
throw new SparkException(
"The ON search condition of the MERGE statement matched a single row from " +
- "the target table with multiple rows of the source table. This could result " +
- "in the target row being operated on more than once with an update or delete " +
- "operation and is not allowed.")
+ "the target table with multiple rows of the source table. This could result " +
+ "in the target row being operated on more than once with an update or delete " +
+ "operation and is not allowed.")
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
index 61d362044c..5dada1cab0 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -24,11 +23,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-case class RenameV2ViewExec(
- catalog: ViewCatalog,
- oldIdent: Identifier,
- newIdent: Identifier) extends LeafV2CommandExec {
+case class RenameV2ViewExec(catalog: ViewCatalog, oldIdent: Identifier, newIdent: Identifier)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
@@ -38,7 +34,6 @@ case class RenameV2ViewExec(
Seq.empty
}
-
override def simpleString(maxFields: Int): String = {
s"RenameV2View ${oldIdent} to {newIdent}"
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala
index 26c652469a..135c194a60 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.expressions.AttributeSet
@@ -26,10 +25,8 @@ import org.apache.spark.sql.execution.SparkPlan
/**
* Physical plan node to replace data in existing tables.
*/
-case class ReplaceDataExec(
- query: SparkPlan,
- refreshCache: () => Unit,
- write: Write) extends V2ExistingTableWriteExec {
+case class ReplaceDataExec(query: SparkPlan, refreshCache: () => Unit, write: Write)
+ extends V2ExistingTableWriteExec {
override lazy val references: AttributeSet = query.outputSet
override lazy val stringArgs: Iterator[Any] = Iterator(query, write)
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
index fcae0a5def..00b998c49e 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -34,7 +33,8 @@ case class ReplacePartitionFieldExec(
ident: Identifier,
transformFrom: Transform,
transformTo: Transform,
- name: Option[String]) extends LeafV2CommandExec {
+ name: Option[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -44,22 +44,26 @@ case class ReplacePartitionFieldExec(
case iceberg: SparkTable =>
val schema = iceberg.table.schema
transformFrom match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
// the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transformFrom))
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transformFrom))
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
}
case table =>
- throw new UnsupportedOperationException(s"Cannot replace partition field in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot replace partition field in non-Iceberg table: $table")
}
Nil
@@ -67,6 +71,6 @@ case class ReplacePartitionFieldExec(
override def simpleString(maxFields: Int): String = {
s"ReplacePartitionField ${catalog.name}.${ident.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala
index 414d4c0ec3..8025f1a072 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
index 8839e730e7..130d1e1bcd 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.expressions.And
@@ -48,15 +47,21 @@ object RowLevelCommandScanRelationPushDown extends Rule[LogicalPlan] with Predic
override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
// use native Spark planning for delta-based plans
// unlike other commands, these plans have filters that can be pushed down directly
- case RewrittenRowLevelCommand(command, _: DataSourceV2Relation, rewritePlan: WriteIcebergDelta) =>
+ case RewrittenRowLevelCommand(
+ command,
+ _: DataSourceV2Relation,
+ rewritePlan: WriteIcebergDelta) =>
val newRewritePlan = V2ScanRelationPushDown.apply(rewritePlan)
command.withNewRewritePlan(newRewritePlan)
// group-based MERGE operations are rewritten as joins and may be planned in a special way
// the join condition is the MERGE condition and can be pushed into the source
// this allows us to remove completely pushed down predicates from the join condition
- case UnplannedGroupBasedMergeOperation(command, rd: ReplaceIcebergData,
- join @ Join(_, _, _, Some(joinCond), _), relation: DataSourceV2Relation) =>
+ case UnplannedGroupBasedMergeOperation(
+ command,
+ rd: ReplaceIcebergData,
+ join @ Join(_, _, _, Some(joinCond), _),
+ relation: DataSourceV2Relation) =>
val table = relation.table.asRowLevelOperationTable
val scanBuilder = table.newScanBuilder(relation.options)
@@ -70,8 +75,7 @@ object RowLevelCommandScanRelationPushDown extends Rule[LogicalPlan] with Predic
val (scan, output) = PushDownUtils.pruneColumns(scanBuilder, relation, relation.output, Nil)
- logInfo(
- s"""
+ logInfo(s"""
|Pushing MERGE operators to ${relation.name}
|Pushed filters: $pushedFiltersStr
|Original JOIN condition: $joinCond
@@ -107,8 +111,7 @@ object RowLevelCommandScanRelationPushDown extends Rule[LogicalPlan] with Predic
val (scan, output) = PushDownUtils.pruneColumns(scanBuilder, relation, relation.output, Nil)
- logInfo(
- s"""
+ logInfo(s"""
|Pushing operators to ${relation.name}
|Pushed filters: $pushedFiltersStr
|Filters that were not pushed: ${remainingFilters.mkString(",")}
@@ -128,7 +131,8 @@ object RowLevelCommandScanRelationPushDown extends Rule[LogicalPlan] with Predic
private def pushFilters(
cond: Expression,
scanBuilder: ScanBuilder,
- tableAttrs: Seq[AttributeReference]): (Either[Seq[Filter], Seq[Predicate]], Seq[Expression]) = {
+ tableAttrs: Seq[AttributeReference])
+ : (Either[Seq[Filter], Seq[Predicate]], Seq[Expression]) = {
val tableAttrSet = AttributeSet(tableAttrs)
val filters = splitConjunctivePredicates(cond).filter(_.references.subsetOf(tableAttrSet))
@@ -178,8 +182,12 @@ object UnplannedGroupBasedMergeOperation {
case rd @ ReplaceIcebergData(DataSourceV2Relation(table, _, _, _, _), query, _, _) =>
val joinsAndRelations = query.collect {
case j @ Join(
- NoStatsUnaryNode(ScanOperation(_, pushDownFilters, pushUpFilters, r: DataSourceV2Relation)), _, _, _, _)
- if pushUpFilters.isEmpty && pushDownFilters.isEmpty && r.table.eq(table) =>
+ NoStatsUnaryNode(
+ ScanOperation(_, pushDownFilters, pushUpFilters, r: DataSourceV2Relation)),
+ _,
+ _,
+ _,
+ _) if pushUpFilters.isEmpty && pushDownFilters.isEmpty && r.table.eq(table) =>
j -> r
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
index b50550ad38..50c53473ab 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -26,10 +25,8 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
import scala.jdk.CollectionConverters._
-case class SetIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
+case class SetIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -37,11 +34,13 @@ case class SetIdentifierFieldsExec(
override protected def run(): Seq[InternalRow] = {
catalog.loadTable(ident) match {
case iceberg: SparkTable =>
- iceberg.table.updateSchema()
+ iceberg.table
+ .updateSchema()
.setIdentifierFields(fields.asJava)
.commit();
case table =>
- throw new UnsupportedOperationException(s"Cannot set identifier fields in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot set identifier fields in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
index c9004ddc5b..9a10949d5e 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.DistributionMode
@@ -36,7 +35,8 @@ case class SetWriteDistributionAndOrderingExec(
catalog: TableCatalog,
ident: Identifier,
distributionMode: Option[DistributionMode],
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafV2CommandExec {
+ sortOrder: Seq[(Term, SortDirection, NullOrder)])
+ extends LeafV2CommandExec {
import CatalogV2Implicits._
@@ -57,7 +57,8 @@ case class SetWriteDistributionAndOrderingExec(
orderBuilder.commit()
distributionMode.foreach { mode =>
- txn.updateProperties()
+ txn
+ .updateProperties()
.set(WRITE_DISTRIBUTION_MODE, mode.modeName())
.commit()
}
@@ -65,7 +66,8 @@ case class SetWriteDistributionAndOrderingExec(
txn.commitTransaction()
case table =>
- throw new UnsupportedOperationException(s"Cannot set write order of non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot set write order of non-Iceberg table: $table")
}
Nil
@@ -73,9 +75,11 @@ case class SetWriteDistributionAndOrderingExec(
override def simpleString(maxFields: Int): String = {
val tableIdent = s"${catalog.name}.${ident.quoted}"
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
+ val order = sortOrder
+ .map { case (term, direction, nullOrder) =>
+ s"$term $direction $nullOrder"
+ }
+ .mkString(", ")
s"SetWriteDistributionAndOrdering $tableIdent $distributionMode $order"
}
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
index ecd3e0d3c9..07ac4aeda8 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -28,7 +27,8 @@ import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
- extends V2CommandExec with LeafExecNode {
+ extends V2CommandExec
+ with LeafExecNode {
override protected def run(): Seq[InternalRow] = {
val builder = new StringBuilder
@@ -43,7 +43,9 @@ case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
private def showColumns(view: View, builder: StringBuilder): Unit = {
val columns = concatByMultiLines(
- view.schema().fields
+ view
+ .schema()
+ .fields
.map(x => s"${x.name}${x.getComment().map(c => s" COMMENT '$c'").getOrElse("")}"))
builder ++= columns
}
@@ -54,14 +56,11 @@ case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
.foreach(builder.append)
}
- private def showProperties(
- view: View,
- builder: StringBuilder): Unit = {
+ private def showProperties(view: View, builder: StringBuilder): Unit = {
val showProps = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
if (showProps.nonEmpty) {
- val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map {
- case (key, value) =>
- s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
+ val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map { case (key, value) =>
+ s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
}
builder ++= "TBLPROPERTIES "
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
index 553b808330..ace43eb6c0 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -26,25 +25,22 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
-case class ShowV2ViewPropertiesExec(
- output: Seq[Attribute],
- view: View,
- propertyKey: Option[String]) extends V2CommandExec with LeafExecNode {
+case class ShowV2ViewPropertiesExec(output: Seq[Attribute], view: View, propertyKey: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
override protected def run(): Seq[InternalRow] = {
propertyKey match {
case Some(p) =>
- val propValue = properties.getOrElse(p,
- s"View ${view.name()} does not have property: $p")
+ val propValue = properties.getOrElse(p, s"View ${view.name()} does not have property: $p")
Seq(toCatalystRow(p, propValue))
case None =>
- properties.map {
- case (k, v) => toCatalystRow(k, v)
+ properties.map { case (k, v) =>
+ toCatalystRow(k, v)
}.toSeq
}
}
-
private def properties = {
view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
}
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
index 3aa85c3db5..83a1f84aec 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -27,10 +26,12 @@ import org.apache.spark.sql.execution.LeafExecNode
import scala.collection.mutable.ArrayBuffer
case class ShowV2ViewsExec(
- output: Seq[Attribute],
- catalog: ViewCatalog,
- namespace: Seq[String],
- pattern: Option[String]) extends V2CommandExec with LeafExecNode {
+ output: Seq[Attribute],
+ catalog: ViewCatalog,
+ namespace: Seq[String],
+ pattern: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -40,7 +41,8 @@ case class ShowV2ViewsExec(
// handle GLOBAL VIEWS
val globalTemp = session.sessionState.catalog.globalTempViewManager.database
if (namespace.nonEmpty && globalTemp == namespace.head) {
- pattern.map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
+ pattern
+ .map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
.getOrElse(session.sessionState.catalog.globalTempViewManager.listViewNames("*"))
.map(name => rows += toCatalystRow(globalTemp, name, true))
} else {
@@ -53,7 +55,8 @@ case class ShowV2ViewsExec(
}
// include TEMP VIEWS
- pattern.map(p => session.sessionState.catalog.listLocalTempViews(p))
+ pattern
+ .map(p => session.sessionState.catalog.listLocalTempViews(p))
.getOrElse(session.sessionState.catalog.listLocalTempViews("*"))
.map(v => rows += toCatalystRow(v.database.toArray.quoted, v.table, true))
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/UpdateRowsExec.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/UpdateRowsExec.scala
index ef4ad37339..9fd0dcd8c0 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/UpdateRowsExec.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/UpdateRowsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.rdd.RDD
@@ -33,7 +32,8 @@ case class UpdateRowsExec(
deleteOutput: Seq[Expression],
insertOutput: Seq[Expression],
output: Seq[Attribute],
- child: SparkPlan) extends UnaryExecNode {
+ child: SparkPlan)
+ extends UnaryExecNode {
@transient override lazy val producedAttributes: AttributeSet = {
AttributeSet(output.filterNot(attr => inputSet.contains(attr)))
@@ -65,7 +65,7 @@ case class UpdateRowsExec(
private val inputRows: Iterator[InternalRow],
private val deleteProj: UnsafeProjection,
private val insertProj: UnsafeProjection)
- extends Iterator[InternalRow] {
+ extends Iterator[InternalRow] {
var cachedInsertRow: InternalRow = _
diff --git a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
index f84bf6bd75..f8acef9fe3 100644
--- a/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
+++ b/spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.dynamicpruning
import org.apache.spark.sql.SparkSession
@@ -58,16 +57,19 @@ import scala.collection.compat.immutable.ArraySeq
* Note that only group-based rewrite plans (i.e. ReplaceData) are taken into account.
* Row-based rewrite plans are subject to usual runtime filtering.
*/
-case class RowLevelCommandDynamicPruning(spark: SparkSession) extends Rule[LogicalPlan] with PredicateHelper {
+case class RowLevelCommandDynamicPruning(spark: SparkSession)
+ extends Rule[LogicalPlan]
+ with PredicateHelper {
import ExtendedDataSourceV2Implicits._
override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown {
// apply special dynamic filtering only for plans that don't support deltas
case RewrittenRowLevelCommand(
- command: RowLevelCommand,
- DataSourceV2ScanRelation(_, scan: SupportsRuntimeFiltering, _, _, _),
- rewritePlan: ReplaceIcebergData) if conf.dynamicPartitionPruningEnabled && isCandidate(command) =>
+ command: RowLevelCommand,
+ DataSourceV2ScanRelation(_, scan: SupportsRuntimeFiltering, _, _, _),
+ rewritePlan: ReplaceIcebergData)
+ if conf.dynamicPartitionPruningEnabled && isCandidate(command) =>
// use reference equality to find exactly the required scan relations
val newRewritePlan = rewritePlan transformUp {
@@ -154,15 +156,15 @@ case class RowLevelCommandDynamicPruning(spark: SparkSession) extends Rule[Logic
}
// borrowed from OptimizeSubqueries in Spark
- private def optimizeSubquery(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
- _.containsPattern(PLAN_EXPRESSION)) {
- case s: SubqueryExpression =>
- val Subquery(newPlan, _) = spark.sessionState.optimizer.execute(Subquery.fromExpression(s))
- // At this point we have an optimized subquery plan that we are going to attach
- // to this subquery expression. Here we can safely remove any top level sort
- // in the plan as tuples produced by a subquery are un-ordered.
- s.withNewPlan(removeTopLevelSort(newPlan))
- }
+ private def optimizeSubquery(plan: LogicalPlan): LogicalPlan =
+ plan.transformAllExpressionsWithPruning(_.containsPattern(PLAN_EXPRESSION)) {
+ case s: SubqueryExpression =>
+ val Subquery(newPlan, _) = spark.sessionState.optimizer.execute(Subquery.fromExpression(s))
+ // At this point we have an optimized subquery plan that we are going to attach
+ // to this subquery expression. Here we can safely remove any top level sort
+ // in the plan as tuples produced by a subquery are un-ordered.
+ s.withNewPlan(removeTopLevelSort(newPlan))
+ }
// borrowed from OptimizeSubqueries in Spark
private def removeTopLevelSort(plan: LogicalPlan): LogicalPlan = {
diff --git a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
index 5acaa6800e..5d5990c23b 100644
--- a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
+++ b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.rdd.PartitionCoalescer
@@ -26,10 +25,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
// this node doesn't extend RepartitionOperation on purpose to keep this logic isolated
// and ignore it in optimizer rules such as CollapseRepartition
-case class OrderAwareCoalesce(
- numPartitions: Int,
- coalescer: PartitionCoalescer,
- child: LogicalPlan) extends OrderPreservingUnaryNode {
+case class OrderAwareCoalesce(numPartitions: Int, coalescer: PartitionCoalescer, child: LogicalPlan)
+ extends OrderPreservingUnaryNode {
override def output: Seq[Attribute] = child.output
diff --git a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
index 7b599eb3da..3ce3d3e82a 100644
--- a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
+++ b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.iceberg.DistributionMode
@@ -29,16 +28,19 @@ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits
case class SetWriteDistributionAndOrdering(
table: Seq[String],
distributionMode: Option[DistributionMode],
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafCommand {
+ sortOrder: Seq[(Term, SortDirection, NullOrder)])
+ extends LeafCommand {
import CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
override def simpleString(maxFields: Int): String = {
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
+ val order = sortOrder
+ .map { case (term, direction, nullOrder) =>
+ s"$term $direction $nullOrder"
+ }
+ .mkString(", ")
s"SetWriteDistributionAndOrdering ${table.quoted} $distributionMode $order"
}
}
diff --git a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
index bf19ef8a21..71b6107d04 100644
--- a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
+++ b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.iceberg.NullOrder
@@ -27,7 +26,9 @@ import org.apache.iceberg.expressions.Term
class SortOrderParserUtil {
- def collectSortOrder(tableSchema:Schema, sortOrder: Seq[(Term, SortDirection, NullOrder)]): SortOrder = {
+ def collectSortOrder(
+ tableSchema: Schema,
+ sortOrder: Seq[(Term, SortDirection, NullOrder)]): SortOrder = {
val orderBuilder = SortOrder.builderFor(tableSchema)
sortOrder.foreach {
case (term, SortDirection.ASC, nullOrder) =>
diff --git a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
index aa9e9c5533..f46127f818 100644
--- a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
+++ b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.utils
import org.apache.iceberg.spark.source.SparkTable
diff --git a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
index 2ef9955052..83dec98c0d 100644
--- a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
+++ b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution
import org.apache.spark.rdd.PartitionCoalescer
@@ -31,7 +30,8 @@ import org.apache.spark.sql.catalyst.plans.physical.UnknownPartitioning
case class OrderAwareCoalesceExec(
numPartitions: Int,
coalescer: PartitionCoalescer,
- child: SparkPlan) extends UnaryExecNode {
+ child: SparkPlan)
+ extends UnaryExecNode {
override def output: Seq[Attribute] = child.output
diff --git a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
index 95a0315e71..c50584be82 100644
--- a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
+++ b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources
import org.apache.iceberg.spark.SparkV2Filters
@@ -32,7 +31,8 @@ import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Strategy
object SparkExpressionConverter {
- def convertToIcebergExpression(sparkExpression: Expression): org.apache.iceberg.expressions.Expression = {
+ def convertToIcebergExpression(
+ sparkExpression: Expression): org.apache.iceberg.expressions.Expression = {
// Currently, it is a double conversion as we are converting Spark expression to Spark predicate
// and then converting Spark predicate to Iceberg expression.
// But these two conversions already exist and well tested. So, we are going with this approach.
@@ -40,26 +40,33 @@ object SparkExpressionConverter {
case Some(filter) =>
val converted = SparkV2Filters.convert(filter)
if (converted == null) {
- throw new IllegalArgumentException(s"Cannot convert Spark filter: $filter to Iceberg expression")
+ throw new IllegalArgumentException(
+ s"Cannot convert Spark filter: $filter to Iceberg expression")
}
converted
case _ =>
- throw new IllegalArgumentException(s"Cannot translate Spark expression: $sparkExpression to data source filter")
+ throw new IllegalArgumentException(
+ s"Cannot translate Spark expression: $sparkExpression to data source filter")
}
}
@throws[AnalysisException]
- def collectResolvedSparkExpression(session: SparkSession, tableName: String, where: String): Expression = {
+ def collectResolvedSparkExpression(
+ session: SparkSession,
+ tableName: String,
+ where: String): Expression = {
val tableAttrs = session.table(tableName).queryExecution.analyzed.output
val unresolvedExpression = session.sessionState.sqlParser.parseExpression(where)
val filter = Filter(unresolvedExpression, DummyRelation(tableAttrs))
val optimizedLogicalPlan = session.sessionState.executePlan(filter).optimizedPlan
- optimizedLogicalPlan.collectFirst {
- case filter: Filter => filter.condition
- case _: DummyRelation => Literal.TrueLiteral
- case _: LocalRelation => Literal.FalseLiteral
- }.getOrElse(throw new AnalysisException("Failed to find filter expression"))
+ optimizedLogicalPlan
+ .collectFirst {
+ case filter: Filter => filter.condition
+ case _: DummyRelation => Literal.TrueLiteral
+ case _: LocalRelation => Literal.FalseLiteral
+ }
+ .getOrElse(throw new AnalysisException("Failed to find filter expression"))
}
case class DummyRelation(output: Seq[Attribute]) extends LeafNode
diff --git a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
index cca16960f4..10e25311df 100644
--- a/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
+++ b/spark/v3.4/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.stats
import java.nio.ByteBuffer
@@ -46,12 +45,13 @@ import org.apache.spark.unsafe.types.UTF8String
* which should be deserialized to Compact sketch before using.
*
* See [[https://iceberg.apache.org/puffin-spec/]] for more information.
- *
*/
case class ThetaSketchAgg(
- child: Expression,
- mutableAggBufferOffset: Int = 0,
- inputAggBufferOffset: Int = 0) extends TypedImperativeAggregate[Sketch] with UnaryLike[Expression] {
+ child: Expression,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends TypedImperativeAggregate[Sketch]
+ with UnaryLike[Expression] {
private lazy val icebergType = SparkSchemaUtil.convert(child.dataType)
@@ -102,7 +102,8 @@ case class ThetaSketchAgg(
CompactSketch.wrap(Memory.wrap(storageFormat))
}
- override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate = {
+ override def withNewMutableAggBufferOffset(
+ newMutableAggBufferOffset: Int): ImperativeAggregate = {
copy(mutableAggBufferOffset = newMutableAggBufferOffset)
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
index aa0d7430a8..54d2f8002e 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.iceberg.spark.extensions
import org.apache.spark.sql.SparkSessionExtensions
@@ -42,12 +41,12 @@ class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
extensions.injectResolutionRule { spark => ResolveViews(spark) }
extensions.injectResolutionRule { _ => ProcedureArgumentCoercion }
extensions.injectCheckRule(_ => CheckViews)
- extensions.injectResolutionRule { _ => RewriteUpdateTableForRowLineage}
- extensions.injectResolutionRule { _ => RewriteMergeIntoTableForRowLineage}
+ extensions.injectResolutionRule { _ => RewriteUpdateTableForRowLineage }
+ extensions.injectResolutionRule { _ => RewriteMergeIntoTableForRowLineage }
// optimizer extensions
extensions.injectOptimizerRule { _ => ReplaceStaticInvoke }
- extensions.injectOptimizerRule { _ => RemoveRowLineageOutputFromOriginalTable}
+ extensions.injectOptimizerRule { _ => RemoveRowLineageOutputFromOriginalTable }
// planner extensions
extensions.injectPlannerStrategy { spark => ExtendedDataSourceV2Strategy(spark) }
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
index 2a0e573b18..aefc672dc8 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -38,23 +37,40 @@ object CheckViews extends (LogicalPlan => Unit) {
override def apply(plan: LogicalPlan): Unit = {
plan foreach {
- case CreateIcebergView(resolvedIdent@ResolvedIdentifier(_: ViewCatalog, _), _, query, columnAliases, _,
- _, _, _, _, replace, _) =>
+ case CreateIcebergView(
+ resolvedIdent @ ResolvedIdentifier(_: ViewCatalog, _),
+ _,
+ query,
+ columnAliases,
+ _,
+ _,
+ _,
+ _,
+ _,
+ replace,
+ _) =>
verifyColumnCount(resolvedIdent, columnAliases, query)
- SchemaUtils.checkColumnNameDuplication(query.schema.fieldNames.toIndexedSeq, SQLConf.get.resolver)
+ SchemaUtils.checkColumnNameDuplication(
+ query.schema.fieldNames.toIndexedSeq,
+ SQLConf.get.resolver)
if (replace) {
- val viewIdent: Seq[String] = resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
+ val viewIdent: Seq[String] =
+ resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
checkCyclicViewReference(viewIdent, query, Seq(viewIdent))
}
case AlterViewAs(ResolvedV2View(_, _), _, _) =>
- throw new AnalysisException("ALTER VIEW <viewName> AS is not supported. Use CREATE OR REPLACE VIEW instead")
+ throw new AnalysisException(
+ "ALTER VIEW <viewName> AS is not supported. Use CREATE OR REPLACE VIEW instead")
case _ => // OK
}
}
- private def verifyColumnCount(ident: ResolvedIdentifier, columns: Seq[String], query: LogicalPlan): Unit = {
+ private def verifyColumnCount(
+ ident: ResolvedIdentifier,
+ columns: Seq[String],
+ query: LogicalPlan): Unit = {
if (columns.nonEmpty) {
if (columns.length > query.output.length) {
throw new AnalysisException(
@@ -75,11 +91,11 @@ object CheckViews extends (LogicalPlan => Unit) {
}
private def checkCyclicViewReference(
- viewIdent: Seq[String],
- plan: LogicalPlan,
- cyclePath: Seq[Seq[String]]): Unit = {
+ viewIdent: Seq[String],
+ plan: LogicalPlan,
+ cyclePath: Seq[Seq[String]]): Unit = {
plan match {
- case sub@SubqueryAlias(_, Project(_, _)) =>
+ case sub @ SubqueryAlias(_, Project(_, _)) =>
val currentViewIdent: Seq[String] = sub.identifier.qualifier :+ sub.identifier.name
checkIfRecursiveView(viewIdent, currentViewIdent, cyclePath, sub.children)
case v1View: View =>
@@ -98,15 +114,17 @@ object CheckViews extends (LogicalPlan => Unit) {
}
private def checkIfRecursiveView(
- viewIdent: Seq[String],
- currentViewIdent: Seq[String],
- cyclePath: Seq[Seq[String]],
- children: Seq[LogicalPlan]
- ): Unit = {
+ viewIdent: Seq[String],
+ currentViewIdent: Seq[String],
+ cyclePath: Seq[Seq[String]],
+ children: Seq[LogicalPlan]): Unit = {
val newCyclePath = cyclePath :+ currentViewIdent
if (currentViewIdent == viewIdent) {
- throw new AnalysisException(String.format("Recursive cycle in view detected: %s (cycle: %s)",
- viewIdent.asIdentifier, newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
+ throw new AnalysisException(
+ String.format(
+ "Recursive cycle in view detected: %s (cycle: %s)",
+ viewIdent.asIdentifier,
+ newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
} else {
children.foreach { c =>
checkCyclicViewReference(viewIdent, c, newCyclePath)
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
index 7f0ca8fadd..aadc332181 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
index ee69b5e344..c2d13bfa02 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import java.util.Locale
@@ -61,7 +60,8 @@ case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with
}
if (duplicateParamNames.nonEmpty) {
- throw new AnalysisException(s"Duplicate parameter names: ${duplicateParamNames.mkString("[", ",", "]")}")
+ throw new AnalysisException(
+ s"Duplicate parameter names: ${duplicateParamNames.mkString("[", ",", "]")}")
}
// optional params should be at the end
@@ -89,7 +89,8 @@ case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with
}
if (missingParamNames.nonEmpty) {
- throw new AnalysisException(s"Missing required parameters: ${missingParamNames.mkString("[", ",", "]")}")
+ throw new AnalysisException(
+ s"Missing required parameters: ${missingParamNames.mkString("[", ",", "]")}")
}
val argExprs = new Array[Expression](params.size)
@@ -141,7 +142,8 @@ case class ResolveProcedures(spark: SparkSession) extends Rule[LogicalPlan] with
}
if (validationErrors.nonEmpty) {
- throw new AnalysisException(s"Could not build name to arg map: ${validationErrors.mkString(", ")}")
+ throw new AnalysisException(
+ s"Could not build name to arg map: ${validationErrors.mkString(", ")}")
}
namedArgs.map(arg => arg.name -> arg).toMap
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
index 0fb5d30a91..1e0e5e8951 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SparkSession
@@ -47,36 +46,52 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case u@UnresolvedRelation(nameParts, _, _)
- if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
+ case u @ UnresolvedRelation(nameParts, _, _)
+ if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
u
- case u@UnresolvedRelation(parts@CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
+ case u @ UnresolvedRelation(parts @ CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
.map(createViewRelation(parts, _))
.getOrElse(u)
- case u@UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
+ case u @ UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
.map(_ => ResolvedV2View(catalog.asViewCatalog, ident))
.getOrElse(u)
- case c@CreateIcebergView(ResolvedIdentifier(_, _), _, query, columnAliases, columnComments, _, _, _, _, _, _)
- if query.resolved && !c.rewritten =>
+ case c @ CreateIcebergView(
+ ResolvedIdentifier(_, _),
+ _,
+ query,
+ columnAliases,
+ columnComments,
+ _,
+ _,
+ _,
+ _,
+ _,
+ _) if query.resolved && !c.rewritten =>
val aliased = aliasColumns(query, columnAliases, columnComments)
- c.copy(query = aliased, queryColumnNames = query.schema.fieldNames.toIndexedSeq, rewritten = true)
+ c.copy(
+ query = aliased,
+ queryColumnNames = query.schema.fieldNames.toIndexedSeq,
+ rewritten = true)
}
private def aliasColumns(
- plan: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]]): LogicalPlan = {
+ plan: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]]): LogicalPlan = {
if (columnAliases.isEmpty || columnAliases.length != plan.output.length) {
plan
} else {
val projectList = plan.output.zipWithIndex.map { case (attr, pos) =>
if (columnComments.apply(pos).isDefined) {
- val meta = new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
+ val meta =
+ new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
Alias(attr, columnAliases.apply(pos))(explicitMetadata = Some(meta))
} else {
Alias(attr, columnAliases.apply(pos))()
@@ -86,7 +101,6 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
}
-
private def createViewRelation(nameParts: Seq[String], view: View): LogicalPlan = {
val parsed = parseViewText(nameParts.quoted, view.query)
@@ -99,17 +113,15 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
// This is more strict because it doesn't allow resolution by field name.
val aliases = view.schema.fields.zipWithIndex.map { case (expected, pos) =>
val attr = GetColumnByOrdinal(pos, expected.dataType)
- Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata = Some(expected.metadata))
+ Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata =
+ Some(expected.metadata))
}.toIndexedSeq
SubqueryAlias(nameParts, Project(aliases, rewritten))
}
private def parseViewText(name: String, viewText: String): LogicalPlan = {
- val origin = Origin(
- objectType = Some("VIEW"),
- objectName = Some(name)
- )
+ val origin = Origin(objectType = Some("VIEW"), objectName = Some(name))
try {
CurrentOrigin.withOrigin(origin) {
@@ -122,8 +134,8 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
private def rewriteIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = {
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = {
// Substitute CTEs and Unresolved Ordinals within the view, then rewrite unresolved functions and relations
qualifyTableIdentifiers(
qualifyFunctionIdentifiers(
@@ -133,15 +145,15 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
private def qualifyFunctionIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
- case u@UnresolvedFunction(Seq(name), _, _, _, _) =>
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
+ case u @ UnresolvedFunction(Seq(name), _, _, _, _) =>
if (!isBuiltinFunction(name)) {
u.copy(nameParts = catalogAndNamespace :+ name)
} else {
u
}
- case u@UnresolvedFunction(parts, _, _, _, _) if !isCatalog(parts.head) =>
+ case u @ UnresolvedFunction(parts, _, _, _, _) if !isCatalog(parts.head) =>
u.copy(nameParts = catalogAndNamespace.head +: parts)
}
@@ -149,17 +161,16 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
* Qualify table identifiers with default catalog and namespace if necessary.
*/
private def qualifyTableIdentifiers(
- child: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan =
+ child: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan =
child transform {
- case u@UnresolvedRelation(Seq(table), _, _) =>
+ case u @ UnresolvedRelation(Seq(table), _, _) =>
u.copy(multipartIdentifier = catalogAndNamespace :+ table)
- case u@UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
+ case u @ UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
u.copy(multipartIdentifier = catalogAndNamespace.head +: parts)
case other =>
- other.transformExpressions {
- case subquery: SubqueryExpression =>
- subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
+ other.transformExpressions { case subquery: SubqueryExpression =>
+ subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
}
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala
index 069caf1a71..39d07c40ea 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTableForRowLineage.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.catalyst.expressions.Literal
@@ -31,9 +30,9 @@ object RewriteMergeIntoTableForRowLineage extends RewriteOperationForRowLineage
override def apply(plan: LogicalPlan): LogicalPlan = {
plan.resolveOperators {
case m @ MergeIntoTable(_, _, _, matchedActions, _, notMatchedBySourceActions)
- if m.resolved && m.rewritable && m.aligned &&
- (matchedActions.nonEmpty || notMatchedBySourceActions.nonEmpty) &&
- shouldUpdatePlan(m.targetTable) =>
+ if m.resolved && m.rewritable && m.aligned &&
+ (matchedActions.nonEmpty || notMatchedBySourceActions.nonEmpty) &&
+ shouldUpdatePlan(m.targetTable) =>
updateMergeIntoForRowLineage(m)
}
}
@@ -47,8 +46,11 @@ object RewriteMergeIntoTableForRowLineage extends RewriteOperationForRowLineage
val matchedAssignmentsForLineage = matchedActions.map {
case UpdateAction(cond, assignments) =>
- UpdateAction(cond, assignments ++ Seq(Assignment(rowId, rowId),
- Assignment(lastUpdatedSequenceNumber, Literal(null))))
+ UpdateAction(
+ cond,
+ assignments ++ Seq(
+ Assignment(rowId, rowId),
+ Assignment(lastUpdatedSequenceNumber, Literal(null))))
case deleteAction => deleteAction
}
@@ -58,8 +60,11 @@ object RewriteMergeIntoTableForRowLineage extends RewriteOperationForRowLineage
// during alignment in ResolveRowLevelCommandAssignments.
val notMatchedBySourceActionsForLineage = notMatchedBySourceActions.map {
case UpdateAction(cond, assignments) =>
- UpdateAction(cond, assignments ++ Seq(Assignment(rowId, rowId),
- Assignment(lastUpdatedSequenceNumber, Literal(null))))
+ UpdateAction(
+ cond,
+ assignments ++ Seq(
+ Assignment(rowId, rowId),
+ Assignment(lastUpdatedSequenceNumber, Literal(null))))
case deleteAction => deleteAction
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala
index 55c2481786..5be30ac606 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteOperationForRowLineage.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.iceberg.MetadataColumns
@@ -32,7 +31,8 @@ import org.apache.spark.sql.types.MetadataBuilder
trait RewriteOperationForRowLineage extends RewriteRowLevelCommand {
protected val ROW_ID_ATTRIBUTE_NAME = MetadataColumns.ROW_ID.name()
- protected val LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name()
+ protected val LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME =
+ MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name()
// The plan should only be updated if the underlying Iceberg table supports row lineage AND
// lineage attributes are not already on the output of operation which indicates the rule already ran
@@ -52,16 +52,17 @@ trait RewriteOperationForRowLineage extends RewriteRowLevelCommand {
}
protected def findRowLineageAttributes(
- expressions: Seq[Expression]
- ): Option[(AttributeReference, AttributeReference)] = {
+ expressions: Seq[Expression]): Option[(AttributeReference, AttributeReference)] = {
val rowIdAttr = expressions.collectFirst {
case attr: AttributeReference
- if isMetadataColumn(attr) && attr.name == ROW_ID_ATTRIBUTE_NAME => attr
+ if isMetadataColumn(attr) && attr.name == ROW_ID_ATTRIBUTE_NAME =>
+ attr
}
val lastUpdatedAttr = expressions.collectFirst {
case attr: AttributeReference
- if isMetadataColumn(attr) && attr.name == LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME => attr
+ if isMetadataColumn(attr) && attr.name == LAST_UPDATED_SEQUENCE_NUMBER_ATTRIBUTE_NAME =>
+ attr
}
// Treat row lineage columns as data columns by removing the metadata attribute
@@ -78,7 +79,8 @@ trait RewriteOperationForRowLineage extends RewriteRowLevelCommand {
attr.withMetadata(
new MetadataBuilder()
.withMetadata(attr.metadata)
- .remove(METADATA_COL_ATTR_KEY).build())
+ .remove(METADATA_COL_ATTR_KEY)
+ .build())
}
private def isMetadataColumn(attributeReference: AttributeReference): Boolean = {
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala
index ea8fd536be..8a8542e100 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteUpdateTableForRowLineage.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.catalyst.expressions.Literal
@@ -30,17 +29,17 @@ object RewriteUpdateTableForRowLineage extends RewriteOperationForRowLineage {
override def apply(plan: LogicalPlan): LogicalPlan = {
plan resolveOperators {
- case updateTable@UpdateTable(_, _, _) if shouldUpdatePlan(updateTable.table) =>
+ case updateTable @ UpdateTable(_, _, _) if shouldUpdatePlan(updateTable.table) =>
updatePlanWithRowLineage(updateTable)
}
}
-
private def updatePlanWithRowLineage(updateTable: UpdateTable): LogicalPlan = {
EliminateSubqueryAliases(updateTable.table) match {
case r @ DataSourceV2Relation(_: SupportsRowLevelOperations, _, _, _, _) =>
val lineageAttributes = findRowLineageAttributes(r.metadataOutput).get
- val (rowId, lastUpdatedSequence) = (removeMetadataColumnAttribute(lineageAttributes._1),
+ val (rowId, lastUpdatedSequence) = (
+ removeMetadataColumnAttribute(lineageAttributes._1),
removeMetadataColumnAttribute(lineageAttributes._2))
val lineageAssignments = updateTable.assignments ++
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
index 800ad1f66a..c47b7d6ef6 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -52,11 +51,19 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case DropView(ResolvedIdent(resolved), ifExists) =>
DropIcebergView(resolved, ifExists)
- case CreateView(ResolvedIdent(resolved), userSpecifiedColumns, comment, properties,
- Some(queryText), query, allowExisting, replace) =>
+ case CreateView(
+ ResolvedIdent(resolved),
+ userSpecifiedColumns,
+ comment,
+ properties,
+ Some(queryText),
+ query,
+ allowExisting,
+ replace) =>
val q = CTESubstitution.apply(query)
verifyTemporaryObjectsDontExist(resolved, q)
- CreateIcebergView(child = resolved,
+ CreateIcebergView(
+ child = resolved,
queryText = queryText,
query = q,
columnAliases = userSpecifiedColumns.map(_._1),
@@ -68,20 +75,25 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case view @ ShowViews(UnresolvedNamespace(Seq()), pattern, output) =>
if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) {
- ShowIcebergViews(ResolvedNamespace(catalogManager.currentCatalog, catalogManager.currentNamespace.toIndexedSeq),
- pattern, output)
+ ShowIcebergViews(
+ ResolvedNamespace(
+ catalogManager.currentCatalog,
+ catalogManager.currentNamespace.toIndexedSeq),
+ pattern,
+ output)
} else {
view
}
case ShowViews(UnresolvedNamespace(CatalogAndNamespace(catalog, ns)), pattern, output)
- if ViewUtil.isViewCatalog(catalog) =>
+ if ViewUtil.isViewCatalog(catalog) =>
ShowIcebergViews(ResolvedNamespace(catalog, ns), pattern, output)
// needs to be done here instead of in ResolveViews, so that a V2 view can be resolved before the Analyzer
// tries to resolve it, which would result in an error, saying that V2 views aren't supported
- case u@UnresolvedView(ResolvedView(resolved), _, _, _) =>
- ViewUtil.loadView(resolved.catalog, resolved.identifier)
+ case u @ UnresolvedView(ResolvedView(resolved), _, _, _) =>
+ ViewUtil
+ .loadView(resolved.catalog, resolved.identifier)
.map(_ => ResolvedV2View(resolved.catalog.asViewCatalog, resolved.identifier))
.getOrElse(u)
}
@@ -102,7 +114,8 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case UnresolvedIdentifier(nameParts, true) if isTempView(nameParts) =>
None
- case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _) if ViewUtil.isViewCatalog(catalog) =>
+ case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _)
+ if ViewUtil.isViewCatalog(catalog) =>
Some(ResolvedIdentifier(catalog, ident))
case _ =>
@@ -114,11 +127,14 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
* Permanent views are not allowed to reference temp objects
*/
private def verifyTemporaryObjectsDontExist(
- identifier: ResolvedIdentifier,
- child: LogicalPlan): Unit = {
+ identifier: ResolvedIdentifier,
+ child: LogicalPlan): Unit = {
val tempViews = collectTemporaryViews(child)
if (tempViews.nonEmpty) {
- throw invalidRefToTempObject(identifier, tempViews.map(v => v.quoted).mkString("[", ", ", "]"), "view")
+ throw invalidRefToTempObject(
+ identifier,
+ tempViews.map(v => v.quoted).mkString("[", ", ", "]"),
+ "view")
}
val tempFunctions = collectTemporaryFunctions(child)
@@ -127,9 +143,17 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
}
}
- private def invalidRefToTempObject(ident: ResolvedIdentifier, tempObjectNames: String, tempObjectType: String) = {
- new AnalysisException(String.format("Cannot create view %s.%s that references temporary %s: %s",
- ident.catalog.name(), ident.identifier, tempObjectType, tempObjectNames))
+ private def invalidRefToTempObject(
+ ident: ResolvedIdentifier,
+ tempObjectNames: String,
+ tempObjectType: String) = {
+ new AnalysisException(
+ String.format(
+ "Cannot create view %s.%s that references temporary %s: %s",
+ ident.catalog.name(),
+ ident.identifier,
+ tempObjectType,
+ tempObjectNames))
}
/**
@@ -141,10 +165,11 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case unresolved: UnresolvedRelation if isTempView(unresolved.multipartIdentifier) =>
Seq(unresolved.multipartIdentifier)
case view: View if view.isTempView => Seq(view.desc.identifier.nameParts)
- case plan => plan.expressions.flatMap(_.flatMap {
- case e: SubqueryExpression => collectTempViews(e.plan)
- case _ => Seq.empty
- })
+ case plan =>
+ plan.expressions.flatMap(_.flatMap {
+ case e: SubqueryExpression => collectTempViews(e.plan)
+ case _ => Seq.empty
+ })
}.distinct
}
@@ -157,7 +182,9 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
None
case CatalogAndIdentifier(catalog, ident) if ViewUtil.isViewCatalog(catalog) =>
- ViewUtil.loadView(catalog, ident).flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
+ ViewUtil
+ .loadView(catalog, ident)
+ .flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
case _ =>
None
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
index d46f10b7f5..a7188837c5 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.connector.catalog.CatalogPlugin
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala
index 70095f95ab..5fd98d03fa 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RemoveRowLineageOutputFromOriginalTable.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.optimizer
import org.apache.iceberg.MetadataColumns
@@ -35,20 +34,19 @@ import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
object RemoveRowLineageOutputFromOriginalTable extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = {
plan.transform {
- case writeDelta@WriteDelta(_, _, _, originalTable, _, _) =>
+ case writeDelta @ WriteDelta(_, _, _, originalTable, _, _) =>
writeDelta.copy(originalTable = removeRowLineageOutput(originalTable))
- case replaceData@ReplaceData(_, _, _, originalTable, _, _) =>
+ case replaceData @ ReplaceData(_, _, _, originalTable, _, _) =>
replaceData.copy(originalTable = removeRowLineageOutput(originalTable))
}
}
private def removeRowLineageOutput(table: NamedRelation): DataSourceV2Relation = {
table match {
- case dsv2Relation@DataSourceV2Relation(_, _, _, _, _) =>
- dsv2Relation.copy(output = dsv2Relation.output.filterNot(
- attr => attr.name == MetadataColumns.ROW_ID.name() ||
+ case dsv2Relation @ DataSourceV2Relation(_, _, _, _, _) =>
+ dsv2Relation.copy(output = dsv2Relation.output.filterNot(attr =>
+ attr.name == MetadataColumns.ROW_ID.name() ||
attr.name == MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name()))
}
}
}
-
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
index d5c4cb84a0..bd22d2ca15 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
@@ -48,7 +48,7 @@ import org.apache.spark.sql.types.StructType
object ReplaceStaticInvoke extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan =
- plan.transformWithPruning (_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
+ plan.transformWithPruning(_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
case replace @ ReplaceData(_, cond, _, _, _, _) =>
replaceStaticInvoke(replace, cond, newCond => replace.copy(condition = newCond))
@@ -57,7 +57,7 @@ object ReplaceStaticInvoke extends Rule[LogicalPlan] {
case filter @ Filter(cond, _) =>
replaceStaticInvoke(filter, cond, newCond => filter.copy(condition = newCond))
- }
+ }
private def replaceStaticInvoke[T <: LogicalPlan](
node: T,
@@ -90,16 +90,17 @@ object ReplaceStaticInvoke extends Rule[LogicalPlan] {
return invoke
}
- val inputType = StructType(invoke.arguments.zipWithIndex.map {
- case (exp, pos) => StructField(s"_$pos", exp.dataType, exp.nullable)
+ val inputType = StructType(invoke.arguments.zipWithIndex.map { case (exp, pos) =>
+ StructField(s"_$pos", exp.dataType, exp.nullable)
})
- val bound = try {
- unbound.bind(inputType)
- } catch {
- case _: Exception =>
- return invoke
- }
+ val bound =
+ try {
+ unbound.bind(inputType)
+ } catch {
+ case _: Exception =>
+ return invoke
+ }
if (bound.inputTypes().length != invoke.arguments.length) {
return invoke
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
index 365a1428f5..b25333d567 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.parser.extensions
import java.util.Locale
@@ -46,7 +45,9 @@ import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.types.StructType
import scala.jdk.CollectionConverters._
-class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserInterface with ExtendedParser {
+class IcebergSparkSqlExtensionsParser(delegate: ParserInterface)
+ extends ParserInterface
+ with ExtendedParser {
import IcebergSparkSqlExtensionsParser._
@@ -115,14 +116,17 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
override def parsePlan(sqlText: String): LogicalPlan = {
val sqlTextAfterSubstitution = substitutor.substitute(sqlText)
if (isIcebergCommand(sqlTextAfterSubstitution)) {
- parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }.asInstanceOf[LogicalPlan]
+ parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }
+ .asInstanceOf[LogicalPlan]
} else {
RewriteViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText))
}
}
private def isIcebergCommand(sqlText: String): Boolean = {
- val normalized = sqlText.toLowerCase(Locale.ROOT).trim()
+ val normalized = sqlText
+ .toLowerCase(Locale.ROOT)
+ .trim()
// Strip simple SQL comments that terminate a line, e.g. comments starting with `--` .
.replaceAll("--.*?\\n", " ")
// Strip newlines.
@@ -134,18 +138,17 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
.replaceAll("`", "")
.trim()
- isIcebergProcedure(normalized) || (
- normalized.startsWith("alter table") && (
- normalized.contains("add partition field") ||
- normalized.contains("drop partition field") ||
- normalized.contains("replace partition field") ||
- normalized.contains("write ordered by") ||
- normalized.contains("write locally ordered by") ||
- normalized.contains("write distributed by") ||
- normalized.contains("write unordered") ||
- normalized.contains("set identifier fields") ||
- normalized.contains("drop identifier fields") ||
- isSnapshotRefDdl(normalized)))
+ isIcebergProcedure(normalized) || (normalized
+ .startsWith("alter table") && (normalized.contains("add partition field") ||
+ normalized.contains("drop partition field") ||
+ normalized.contains("replace partition field") ||
+ normalized.contains("write ordered by") ||
+ normalized.contains("write locally ordered by") ||
+ normalized.contains("write distributed by") ||
+ normalized.contains("write unordered") ||
+ normalized.contains("set identifier fields") ||
+ normalized.contains("drop identifier fields") ||
+ isSnapshotRefDdl(normalized)))
}
// All builtin Iceberg procedures are under the 'system' namespace
@@ -156,15 +159,16 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
private def isSnapshotRefDdl(normalized: String): Boolean = {
normalized.contains("create branch") ||
- normalized.contains("replace branch") ||
- normalized.contains("create tag") ||
- normalized.contains("replace tag") ||
- normalized.contains("drop branch") ||
- normalized.contains("drop tag")
+ normalized.contains("replace branch") ||
+ normalized.contains("create tag") ||
+ normalized.contains("replace tag") ||
+ normalized.contains("drop branch") ||
+ normalized.contains("drop tag")
}
protected def parse[T](command: String)(toResult: IcebergSqlExtensionsParser => T): T = {
- val lexer = new IcebergSqlExtensionsLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
+ val lexer = new IcebergSqlExtensionsLexer(
+ new UpperCaseCharStream(CharStreams.fromString(command)))
lexer.removeErrorListeners()
lexer.addErrorListener(IcebergParseErrorListener)
@@ -182,8 +186,7 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
parser.setErrorHandler(new BailErrorStrategy)
parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
toResult(parser)
- }
- catch {
+ } catch {
case _: ParseCancellationException =>
// if we fail, parse with LL mode with DefaultErrorStrategy
tokenStream.seek(0) // rewind input stream
@@ -194,8 +197,7 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
parser.getInterpreter.setPredictionMode(PredictionMode.LL)
toResult(parser)
}
- }
- catch {
+ } catch {
case e: IcebergParseException if e.command.isDefined =>
throw e
case e: IcebergParseException =>
@@ -213,7 +215,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
object IcebergSparkSqlExtensionsParser {
private val substitutorCtor: DynConstructors.Ctor[VariableSubstitution] =
- DynConstructors.builder()
+ DynConstructors
+ .builder()
.impl(classOf[VariableSubstitution])
.impl(classOf[VariableSubstitution], classOf[SQLConf])
.build()
@@ -259,9 +262,7 @@ case object IcebergSqlExtensionsPostProcessor extends IcebergSqlExtensionsBaseLi
replaceTokenByIdentifier(ctx, 0)(identity)
}
- private def replaceTokenByIdentifier(
- ctx: ParserRuleContext,
- stripMargins: Int)(
+ private def replaceTokenByIdentifier(ctx: ParserRuleContext, stripMargins: Int)(
f: CommonToken => CommonToken = identity): Unit = {
val parent = ctx.getParent
parent.removeLastChild()
@@ -308,10 +309,12 @@ class IcebergParseException(
val command: Option[String],
message: String,
val start: Origin,
- val stop: Origin) extends AnalysisException(message, start.line, start.startPosition) {
+ val stop: Origin)
+ extends AnalysisException(message, start.line, start.startPosition) {
def this(message: String, ctx: ParserRuleContext) = {
- this(Option(IcebergParserUtils.command(ctx)),
+ this(
+ Option(IcebergParserUtils.command(ctx)),
message,
IcebergParserUtils.position(ctx.getStart),
IcebergParserUtils.position(ctx.getStop))
@@ -321,8 +324,7 @@ class IcebergParseException(
val builder = new StringBuilder
builder ++= "\n" ++= message
start match {
- case Origin(
- Some(l), Some(p), Some(_), Some(_), Some(_), Some(_), Some(_)) =>
+ case Origin(Some(l), Some(p), Some(_), Some(_), Some(_), Some(_), Some(_)) =>
builder ++= s"(line $l, pos $p)\n"
command.foreach { cmd =>
val (above, below) = cmd.split("\n").splitAt(l)
@@ -342,4 +344,4 @@ class IcebergParseException(
def withCommand(cmd: String): IcebergParseException = {
new IcebergParseException(Option(cmd), message, start, stop)
}
-}
\ No newline at end of file
+}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
index 6b1cc41da0..f87733e6b7 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.parser.extensions
import java.util.Locale
@@ -63,9 +62,11 @@ import org.apache.spark.sql.connector.expressions.LiteralValue
import org.apache.spark.sql.connector.expressions.Transform
import scala.jdk.CollectionConverters._
-class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
+class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface)
+ extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
- private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] = list.asScala
+ private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] =
+ list.asScala
private def toSeq[T](list: java.util.List[T]): Seq[T] = toBuffer(list).toSeq
/**
@@ -80,52 +81,59 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an ADD PARTITION FIELD logical command.
*/
- override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField = withOrigin(ctx) {
- AddPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform),
- Option(ctx.name).map(_.getText))
- }
+ override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField =
+ withOrigin(ctx) {
+ AddPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform),
+ Option(ctx.name).map(_.getText))
+ }
/**
* Create a DROP PARTITION FIELD logical command.
*/
- override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField = withOrigin(ctx) {
- DropPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform))
- }
+ override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField =
+ withOrigin(ctx) {
+ DropPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform))
+ }
/**
* Create a CREATE OR REPLACE BRANCH logical command.
*/
- override def visitCreateOrReplaceBranch(ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
+ override def visitCreateOrReplaceBranch(
+ ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
val createOrReplaceBranchClause = ctx.createReplaceBranchClause()
val branchName = createOrReplaceBranchClause.identifier()
val branchOptionsContext = Option(createOrReplaceBranchClause.branchOptions())
- val snapshotId = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotId()))
+ val snapshotId = branchOptionsContext
+ .flatMap(branchOptions => Option(branchOptions.snapshotId()))
.map(_.getText.toLong)
- val snapshotRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
- val minSnapshotsToKeep = snapshotRetention.flatMap(retention => Option(retention.minSnapshotsToKeep()))
+ val snapshotRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
+ val minSnapshotsToKeep = snapshotRetention
+ .flatMap(retention => Option(retention.minSnapshotsToKeep()))
.map(minSnapshots => minSnapshots.number().getText.toLong)
val maxSnapshotAgeMs = snapshotRetention
.flatMap(retention => Option(retention.maxSnapshotAge()))
- .map(retention => TimeUnit.valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
- .toMillis(retention.number().getText.toLong))
- val branchRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
+ .map(retention =>
+ TimeUnit
+ .valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retention.number().getText.toLong))
+ val branchRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
val branchRefAgeMs = branchRetention.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
val create = createOrReplaceBranchClause.CREATE() != null
val replace = ctx.createReplaceBranchClause().REPLACE() != null
val ifNotExists = createOrReplaceBranchClause.EXISTS() != null
- val branchOptions = BranchOptions(
- snapshotId,
- minSnapshotsToKeep,
- maxSnapshotAgeMs,
- branchRefAgeMs
- )
+ val branchOptions =
+ BranchOptions(snapshotId, minSnapshotsToKeep, maxSnapshotAgeMs, branchRefAgeMs)
CreateOrReplaceBranch(
typedVisit[Seq[String]](ctx.multipartIdentifier),
@@ -139,52 +147,61 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an CREATE OR REPLACE TAG logical command.
*/
- override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag = withOrigin(ctx) {
- val createTagClause = ctx.createReplaceTagClause()
-
- val tagName = createTagClause.identifier().getText
-
- val tagOptionsContext = Option(createTagClause.tagOptions())
- val snapshotId = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.snapshotId()))
- .map(_.getText.toLong)
- val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
- val tagRefAgeMs = tagRetain.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
- val tagOptions = TagOptions(
- snapshotId,
- tagRefAgeMs
- )
-
- val create = createTagClause.CREATE() != null
- val replace = createTagClause.REPLACE() != null
- val ifNotExists = createTagClause.EXISTS() != null
-
- CreateOrReplaceTag(typedVisit[Seq[String]](ctx.multipartIdentifier),
- tagName,
- tagOptions,
- create,
- replace,
- ifNotExists)
- }
+ override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag =
+ withOrigin(ctx) {
+ val createTagClause = ctx.createReplaceTagClause()
+
+ val tagName = createTagClause.identifier().getText
+
+ val tagOptionsContext = Option(createTagClause.tagOptions())
+ val snapshotId = tagOptionsContext
+ .flatMap(tagOptions => Option(tagOptions.snapshotId()))
+ .map(_.getText.toLong)
+ val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
+ val tagRefAgeMs = tagRetain.map(retain =>
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
+ val tagOptions = TagOptions(snapshotId, tagRefAgeMs)
+
+ val create = createTagClause.CREATE() != null
+ val replace = createTagClause.REPLACE() != null
+ val ifNotExists = createTagClause.EXISTS() != null
+
+ CreateOrReplaceTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ tagName,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists)
+ }
/**
* Create an DROP BRANCH logical command.
*/
override def visitDropBranch(ctx: DropBranchContext): DropBranch = withOrigin(ctx) {
- DropBranch(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
+ DropBranch(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
}
/**
* Create an DROP TAG logical command.
*/
override def visitDropTag(ctx: DropTagContext): DropTag = withOrigin(ctx) {
- DropTag(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
+ DropTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
}
/**
* Create an REPLACE PARTITION FIELD logical command.
*/
- override def visitReplacePartitionField(ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
+ override def visitReplacePartitionField(
+ ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
ReplacePartitionField(
typedVisit[Seq[String]](ctx.multipartIdentifier),
typedVisit[Transform](ctx.transform(0)),
@@ -195,20 +212,22 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an SET IDENTIFIER FIELDS logical command.
*/
- override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields = withOrigin(ctx) {
- SetIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
+ override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields =
+ withOrigin(ctx) {
+ SetIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
/**
* Create an DROP IDENTIFIER FIELDS logical command.
*/
- override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields = withOrigin(ctx) {
- DropIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
+ override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields =
+ withOrigin(ctx) {
+ DropIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
/**
* Create a [[SetWriteDistributionAndOrdering]] for changing the write distribution and ordering.
@@ -266,12 +285,15 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
*/
override def visitOrderField(ctx: OrderFieldContext): (Term, SortDirection, NullOrder) = {
val term = Spark3Util.toIcebergTerm(typedVisit[Transform](ctx.transform))
- val direction = Option(ctx.ASC).map(_ => SortDirection.ASC)
- .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
- .getOrElse(SortDirection.ASC)
- val nullOrder = Option(ctx.FIRST).map(_ => NullOrder.NULLS_FIRST)
- .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
- .getOrElse(if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
+ val direction = Option(ctx.ASC)
+ .map(_ => SortDirection.ASC)
+ .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
+ .getOrElse(SortDirection.ASC)
+ val nullOrder = Option(ctx.FIRST)
+ .map(_ => NullOrder.NULLS_FIRST)
+ .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
+ .getOrElse(
+ if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
(term, direction, nullOrder)
}
@@ -293,35 +315,40 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create a transform argument from a column reference or a constant.
*/
- override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression = withOrigin(ctx) {
- val reference = Option(ctx.multipartIdentifier())
+ override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression =
+ withOrigin(ctx) {
+ val reference = Option(ctx.multipartIdentifier())
.map(typedVisit[Seq[String]])
.map(FieldReference(_))
- val literal = Option(ctx.constant)
+ val literal = Option(ctx.constant)
.map(visitConstant)
.map(lit => LiteralValue(lit.value, lit.dataType))
- reference.orElse(literal)
+ reference
+ .orElse(literal)
.getOrElse(throw new IcebergParseException(s"Invalid transform argument", ctx))
- }
+ }
/**
* Return a multi-part identifier as Seq[String].
*/
- override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) {
- toSeq(ctx.parts).map(_.getText)
- }
+ override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] =
+ withOrigin(ctx) {
+ toSeq(ctx.parts).map(_.getText)
+ }
- override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] = withOrigin(ctx) {
- toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
- }
+ override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] =
+ withOrigin(ctx) {
+ toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
+ }
/**
* Create a positional argument in a stored procedure call.
*/
- override def visitPositionalArgument(ctx: PositionalArgumentContext): CallArgument = withOrigin(ctx) {
- val expr = typedVisit[Expression](ctx.expression)
- PositionalArgument(expr)
- }
+ override def visitPositionalArgument(ctx: PositionalArgumentContext): CallArgument =
+ withOrigin(ctx) {
+ val expr = typedVisit[Expression](ctx.expression)
+ PositionalArgument(expr)
+ }
/**
* Create a named argument in a stored procedure call.
@@ -350,10 +377,12 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
}
private def reconstructSqlString(ctx: ParserRuleContext): String = {
- toBuffer(ctx.children).map {
- case c: ParserRuleContext => reconstructSqlString(c)
- case t: TerminalNode => t.getText
- }.mkString(" ")
+ toBuffer(ctx.children)
+ .map {
+ case c: ParserRuleContext => reconstructSqlString(c)
+ case t: TerminalNode => t.getText
+ }
+ .mkString(" ")
}
private def typedVisit[T](ctx: ParseTree): T = {
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
index e8b1b29411..0a830dbd4f 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
@@ -16,13 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.expressions.Transform
-case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String]) extends LeafCommand {
+case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String])
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
index 4d7e0a086b..15b9083002 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
@@ -16,8 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
-case class BranchOptions (snapshotId: Option[Long], numSnapshots: Option[Long],
- snapshotRetain: Option[Long], snapshotRefRetain: Option[Long])
+case class BranchOptions(
+ snapshotId: Option[Long],
+ numSnapshots: Option[Long],
+ snapshotRetain: Option[Long],
+ snapshotRefRetain: Option[Long])
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
index 9e3fdb0e9e..7fb871bafd 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
index b7981a3c7a..6900f6e8cc 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -27,7 +26,8 @@ case class CreateOrReplaceBranch(
branchOptions: BranchOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
+ ifNotExists: Boolean)
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
index 6e7db84a90..957c68e7a5 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -27,7 +26,8 @@ case class CreateOrReplaceTag(
tagOptions: TagOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
+ ifNotExists: Boolean)
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
index bee0b0fae6..ed4f1f512b 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
index 29dd686a0f..1a91806280 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
@@ -16,14 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
-case class DropIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
+case class DropIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
index fb14513241..ec952c8c71 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
index 7e4b38e74d..da69ca0383 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
index 8c660c6f37..c2525369e7 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -26,13 +25,14 @@ case class ReplacePartitionField(
table: Seq[String],
transformFrom: Transform,
transformTo: Transform,
- name: Option[String]) extends LeafCommand {
+ name: Option[String])
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
override def simpleString(maxFields: Int): String = {
s"ReplacePartitionField ${table.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
}
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
index 1e5622e9c3..8cd2c0ddad 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
@@ -16,14 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
-case class SetIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
+case class SetIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
index 85e3b95f4a..6afe1478d7 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
case class TagOptions(snapshotId: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index be15f32bc1..9986b30c45 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Expression
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
index 9366d5efe1..c5dadae701 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
@@ -16,29 +16,30 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.plans.logical.BinaryCommand
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
case class CreateIcebergView(
- child: LogicalPlan,
- queryText: String,
- query: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String] = Seq.empty,
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean,
- rewritten: Boolean = false) extends BinaryCommand {
+ child: LogicalPlan,
+ queryText: String,
+ query: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String] = Seq.empty,
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean,
+ rewritten: Boolean = false)
+ extends BinaryCommand {
override def left: LogicalPlan = child
override def right: LogicalPlan = query
override protected def withNewChildrenInternal(
- newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+ newLeft: LogicalPlan,
+ newRight: LogicalPlan): LogicalPlan =
copy(child = newLeft, query = newRight)
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
index 275dba6fbf..092b6b33fb 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
@@ -16,15 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
-case class DropIcebergView(
- child: LogicalPlan,
- ifExists: Boolean) extends UnaryCommand {
+case class DropIcebergView(child: LogicalPlan, ifExists: Boolean) extends UnaryCommand {
override protected def withNewChildInternal(newChild: LogicalPlan): DropIcebergView =
copy(child = newChild)
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
index b9c05ff006..4d384e8577 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.analysis.LeafNodeWithoutStats
@@ -24,8 +23,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-case class ResolvedV2View(
- catalog: ViewCatalog,
- identifier: Identifier) extends LeafNodeWithoutStats {
+case class ResolvedV2View(catalog: ViewCatalog, identifier: Identifier)
+ extends LeafNodeWithoutStats {
override def output: Seq[Attribute] = Nil
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
index b09c27acdc..cbfe23d94c 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
@@ -16,8 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -26,9 +24,10 @@ import org.apache.spark.sql.catalyst.plans.logical.ShowViews
import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
case class ShowIcebergViews(
- namespace: LogicalPlan,
- pattern: Option[String],
- override val output: Seq[Attribute] = ShowViews.getOutputAttrs) extends UnaryCommand {
+ namespace: LogicalPlan,
+ pattern: Option[String],
+ override val output: Seq[Attribute] = ShowViews.getOutputAttrs)
+ extends UnaryCommand {
override def child: LogicalPlan = namespace
override protected def withNewChildInternal(newChild: LogicalPlan): ShowIcebergViews =
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
index 55f327f7e4..e28dcfb194 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -31,7 +30,8 @@ case class AddPartitionFieldExec(
catalog: TableCatalog,
ident: Identifier,
transform: Transform,
- name: Option[String]) extends LeafV2CommandExec {
+ name: Option[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -39,12 +39,14 @@ case class AddPartitionFieldExec(
override protected def run(): Seq[InternalRow] = {
catalog.loadTable(ident) match {
case iceberg: SparkTable =>
- iceberg.table.updateSpec()
- .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
+ .commit()
case table =>
- throw new UnsupportedOperationException(s"Cannot add partition field to non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot add partition field to non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
index b103d1ee2c..d6630e51ff 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,17 +24,17 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.connector.catalog.ViewChange
-
case class AlterV2ViewSetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- properties: Map[String, String]) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ properties: Map[String, String])
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
override protected def run(): Seq[InternalRow] = {
- val changes = properties.map {
- case (property, value) => ViewChange.setProperty(property, value)
+ val changes = properties.map { case (property, value) =>
+ ViewChange.setProperty(property, value)
}.toSeq
catalog.alterView(ident, changes: _*)
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
index 91abd95f57..7ebf0b9f70 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.AnalysisException
@@ -26,12 +25,12 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.connector.catalog.ViewChange
-
case class AlterV2ViewUnsetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- propertyKeys: Seq[String],
- ifExists: Boolean) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ propertyKeys: Seq[String],
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
index f66962a8c4..a122775025 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,10 +24,8 @@ import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.connector.iceberg.catalog.Procedure
import scala.collection.compat.immutable.ArraySeq
-case class CallExec(
- output: Seq[Attribute],
- procedure: Procedure,
- input: InternalRow) extends LeafV2CommandExec {
+case class CallExec(output: Seq[Attribute], procedure: Procedure, input: InternalRow)
+ extends LeafV2CommandExec {
override protected def run(): Seq[InternalRow] = {
ArraySeq.unsafeWrapArray(procedure.call(input))
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
index 2be406e7f3..baf985f53a 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -28,13 +27,14 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
case class CreateOrReplaceBranchExec(
- catalog: TableCatalog,
- ident: Identifier,
- branch: String,
- branchOptions: BranchOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
+ catalog: TableCatalog,
+ ident: Identifier,
+ branch: String,
+ branchOptions: BranchOptions,
+ create: Boolean,
+ replace: Boolean,
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -62,8 +62,10 @@ case class CreateOrReplaceBranchExec(
if (create && replace && !refExists) {
safeCreateBranch()
} else if (replace) {
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete replace branch operation on %s, main has no snapshot", ident)
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete replace branch operation on %s, main has no snapshot",
+ ident)
manageSnapshots.replaceBranch(branch, snapshotId)
} else {
if (refExists && ifNotExists) {
@@ -88,7 +90,8 @@ case class CreateOrReplaceBranchExec(
manageSnapshots.commit()
case table =>
- throw new UnsupportedOperationException(s"Cannot create or replace branch on non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot create or replace branch on non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
index 372cd75486..e486892614 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -33,7 +32,8 @@ case class CreateOrReplaceTagExec(
tagOptions: TagOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -47,8 +47,10 @@ case class CreateOrReplaceTagExec(
.map(java.lang.Long.valueOf)
.orNull
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete create or replace tag operation on %s, main has no snapshot", ident)
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete create or replace tag operation on %s, main has no snapshot",
+ ident)
val manageSnapshot = iceberg.table.manageSnapshots()
val refExists = null != iceberg.table().refs().get(tag)
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
index eb4cc7de48..c9d4047658 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.SupportsReplaceView
@@ -29,31 +28,33 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.types.StructType
import scala.jdk.CollectionConverters._
-
case class CreateV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- queryText: String,
- viewSchema: StructType,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String],
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ queryText: String,
+ viewSchema: StructType,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String],
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
override protected def run(): Seq[InternalRow] = {
val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name
- val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
+ val currentCatalog =
+ if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
val currentNamespace = session.sessionState.catalogManager.currentNamespace
val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION
val newProperties = properties ++
comment.map(ViewCatalog.PROP_COMMENT -> _) ++
- Map(ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
+ Map(
+ ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
ViewCatalog.PROP_ENGINE_VERSION -> engineVersion)
if (replace) {
@@ -87,10 +88,10 @@ case class CreateV2ViewExec(
}
private def replaceView(
- supportsReplaceView: SupportsReplaceView,
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
+ supportsReplaceView: SupportsReplaceView,
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
supportsReplaceView.replaceView(
ident,
queryText,
@@ -104,9 +105,9 @@ case class CreateV2ViewExec(
}
private def createView(
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
catalog.createView(
ident,
queryText,
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
index a9c7312d4a..f25ac8e3c3 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -27,10 +26,9 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
-case class DescribeV2ViewExec(
- output: Seq[Attribute],
- view: View,
- isExtended: Boolean) extends V2CommandExec with LeafExecNode {
+case class DescribeV2ViewExec(output: Seq[Attribute], view: View, isExtended: Boolean)
+ extends V2CommandExec
+ with LeafExecNode {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -44,23 +42,22 @@ case class DescribeV2ViewExec(
private def describeSchema: Seq[InternalRow] =
view.schema().map { column =>
- toCatalystRow(
- column.name,
- column.dataType.simpleString,
- column.getComment().getOrElse(""))
+ toCatalystRow(column.name, column.dataType.simpleString, column.getComment().getOrElse(""))
}
private def emptyRow: InternalRow = toCatalystRow("", "", "")
private def describeExtended: Seq[InternalRow] = {
val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
- val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
+ val properties: Map[String, String] =
+ view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2)
- val viewProperties = properties.toSeq.sortBy(_._1).map {
- case (key, value) =>
+ val viewProperties = properties.toSeq
+ .sortBy(_._1)
+ .map { case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
- }.mkString("[", ", ", "]")
-
+ }
+ .mkString("[", ", ", "]")
// omitting view text here because it is shown as
// part of SHOW CREATE TABLE and can result in weird formatting in the DESCRIBE output
@@ -69,7 +66,10 @@ case class DescribeV2ViewExec(
toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
toCatalystRow("View Query Output Columns", outputColumns, "") ::
toCatalystRow("View Properties", viewProperties, "") ::
- toCatalystRow("Created By", view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""), "") ::
+ toCatalystRow(
+ "Created By",
+ view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""),
+ "") ::
Nil
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
index ff8f182009..e7d9c7b70d 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -29,7 +28,8 @@ case class DropBranchExec(
catalog: TableCatalog,
ident: Identifier,
branch: String,
- ifExists: Boolean) extends LeafV2CommandExec {
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
index dee778b474..87b18594d5 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -27,10 +26,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
-case class DropIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
+case class DropIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -42,18 +39,24 @@ case class DropIdentifierFieldsExec(
val identifierFieldNames = Sets.newHashSet(schema.identifierFieldNames)
for (name <- fields) {
- Preconditions.checkArgument(schema.findField(name) != null,
- "Cannot complete drop identifier fields operation: field %s not found", name)
- Preconditions.checkArgument(identifierFieldNames.contains(name),
- "Cannot complete drop identifier fields operation: %s is not an identifier field", name)
+ Preconditions.checkArgument(
+ schema.findField(name) != null,
+ "Cannot complete drop identifier fields operation: field %s not found",
+ name)
+ Preconditions.checkArgument(
+ identifierFieldNames.contains(name),
+ "Cannot complete drop identifier fields operation: %s is not an identifier field",
+ name)
identifierFieldNames.remove(name)
}
- iceberg.table.updateSchema()
+ iceberg.table
+ .updateSchema()
.setIdentifierFields(identifierFieldNames)
.commit();
case table =>
- throw new UnsupportedOperationException(s"Cannot drop identifier fields in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot drop identifier fields in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
index 9a153f0c00..db43263e0e 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -29,10 +28,8 @@ import org.apache.spark.sql.connector.expressions.FieldReference
import org.apache.spark.sql.connector.expressions.IdentityTransform
import org.apache.spark.sql.connector.expressions.Transform
-case class DropPartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transform: Transform) extends LeafV2CommandExec {
+case class DropPartitionFieldExec(catalog: TableCatalog, ident: Identifier, transform: Transform)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -42,20 +39,24 @@ case class DropPartitionFieldExec(
case iceberg: SparkTable =>
val schema = iceberg.table.schema
transform match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
// the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .commit()
case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transform))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transform))
+ .commit()
}
case table =>
- throw new UnsupportedOperationException(s"Cannot drop partition field in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot drop partition field in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
index 8df88765a9..79b4a15255 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -25,11 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
-case class DropTagExec(
- catalog: TableCatalog,
- ident: Identifier,
- tag: String,
- ifExists: Boolean) extends LeafV2CommandExec {
+case class DropTagExec(catalog: TableCatalog, ident: Identifier, tag: String, ifExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
index c35af1486f..6dd1188b78 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,11 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-case class DropV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- ifExists: Boolean) extends LeafV2CommandExec {
+case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
index 6a4dc44608..64b228a424 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -72,11 +71,28 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
AddPartitionFieldExec(catalog, ident, transform, name) :: Nil
case CreateOrReplaceBranch(
- IcebergCatalogAndIdentifier(catalog, ident), branch, branchOptions, create, replace, ifNotExists) =>
- CreateOrReplaceBranchExec(catalog, ident, branch, branchOptions, create, replace, ifNotExists) :: Nil
+ IcebergCatalogAndIdentifier(catalog, ident),
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) =>
+ CreateOrReplaceBranchExec(
+ catalog,
+ ident,
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) :: Nil
case CreateOrReplaceTag(
- IcebergCatalogAndIdentifier(catalog, ident), tag, tagOptions, create, replace, ifNotExists) =>
+ IcebergCatalogAndIdentifier(catalog, ident),
+ tag,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists) =>
CreateOrReplaceTagExec(catalog, ident, tag, tagOptions, create, replace, ifNotExists) :: Nil
case DropBranch(IcebergCatalogAndIdentifier(catalog, ident), branch, ifExists) =>
@@ -88,7 +104,11 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
case DropPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform) =>
DropPartitionFieldExec(catalog, ident, transform) :: Nil
- case ReplacePartitionField(IcebergCatalogAndIdentifier(catalog, ident), transformFrom, transformTo, name) =>
+ case ReplacePartitionField(
+ IcebergCatalogAndIdentifier(catalog, ident),
+ transformFrom,
+ transformTo,
+ name) =>
ReplacePartitionFieldExec(catalog, ident, transformFrom, transformTo, name) :: Nil
case SetIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
@@ -98,13 +118,15 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
DropIdentifierFieldsExec(catalog, ident, fields) :: Nil
case SetWriteDistributionAndOrdering(
- IcebergCatalogAndIdentifier(catalog, ident), distributionMode, ordering) =>
+ IcebergCatalogAndIdentifier(catalog, ident),
+ distributionMode,
+ ordering) =>
SetWriteDistributionAndOrderingExec(catalog, ident, distributionMode, ordering) :: Nil
case OrderAwareCoalesce(numPartitions, coalescer, child) =>
OrderAwareCoalesceExec(numPartitions, coalescer, planLater(child)) :: Nil
- case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView@true) =>
+ case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView @ true) =>
val newIdent = Spark3Util.catalogAndIdentifier(spark, newName.toList.asJava)
if (oldCatalog.name != newIdent.catalog().name()) {
throw new AnalysisException(
@@ -115,8 +137,18 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
case DropIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), ifExists) =>
DropV2ViewExec(viewCatalog, ident, ifExists) :: Nil
- case CreateIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), queryText, query,
- columnAliases, columnComments, queryColumnNames, comment, properties, allowExisting, replace, _) =>
+ case CreateIcebergView(
+ ResolvedIdentifier(viewCatalog: ViewCatalog, ident),
+ queryText,
+ query,
+ columnAliases,
+ columnComments,
+ queryColumnNames,
+ comment,
+ properties,
+ allowExisting,
+ replace,
+ _) =>
CreateV2ViewExec(
catalog = viewCatalog,
ident = ident,
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
index 61d362044c..5dada1cab0 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -24,11 +23,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-case class RenameV2ViewExec(
- catalog: ViewCatalog,
- oldIdent: Identifier,
- newIdent: Identifier) extends LeafV2CommandExec {
+case class RenameV2ViewExec(catalog: ViewCatalog, oldIdent: Identifier, newIdent: Identifier)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
@@ -38,7 +34,6 @@ case class RenameV2ViewExec(
Seq.empty
}
-
override def simpleString(maxFields: Int): String = {
s"RenameV2View ${oldIdent} to {newIdent}"
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
index fcae0a5def..00b998c49e 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -34,7 +33,8 @@ case class ReplacePartitionFieldExec(
ident: Identifier,
transformFrom: Transform,
transformTo: Transform,
- name: Option[String]) extends LeafV2CommandExec {
+ name: Option[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -44,22 +44,26 @@ case class ReplacePartitionFieldExec(
case iceberg: SparkTable =>
val schema = iceberg.table.schema
transformFrom match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
// the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transformFrom))
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transformFrom))
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
}
case table =>
- throw new UnsupportedOperationException(s"Cannot replace partition field in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot replace partition field in non-Iceberg table: $table")
}
Nil
@@ -67,6 +71,6 @@ case class ReplacePartitionFieldExec(
override def simpleString(maxFields: Int): String = {
s"ReplacePartitionField ${catalog.name}.${ident.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
}
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
index b50550ad38..50c53473ab 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -26,10 +25,8 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
import scala.jdk.CollectionConverters._
-case class SetIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
+case class SetIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -37,11 +34,13 @@ case class SetIdentifierFieldsExec(
override protected def run(): Seq[InternalRow] = {
catalog.loadTable(ident) match {
case iceberg: SparkTable =>
- iceberg.table.updateSchema()
+ iceberg.table
+ .updateSchema()
.setIdentifierFields(fields.asJava)
.commit();
case table =>
- throw new UnsupportedOperationException(s"Cannot set identifier fields in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot set identifier fields in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
index c9004ddc5b..9a10949d5e 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.DistributionMode
@@ -36,7 +35,8 @@ case class SetWriteDistributionAndOrderingExec(
catalog: TableCatalog,
ident: Identifier,
distributionMode: Option[DistributionMode],
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafV2CommandExec {
+ sortOrder: Seq[(Term, SortDirection, NullOrder)])
+ extends LeafV2CommandExec {
import CatalogV2Implicits._
@@ -57,7 +57,8 @@ case class SetWriteDistributionAndOrderingExec(
orderBuilder.commit()
distributionMode.foreach { mode =>
- txn.updateProperties()
+ txn
+ .updateProperties()
.set(WRITE_DISTRIBUTION_MODE, mode.modeName())
.commit()
}
@@ -65,7 +66,8 @@ case class SetWriteDistributionAndOrderingExec(
txn.commitTransaction()
case table =>
- throw new UnsupportedOperationException(s"Cannot set write order of non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot set write order of non-Iceberg table: $table")
}
Nil
@@ -73,9 +75,11 @@ case class SetWriteDistributionAndOrderingExec(
override def simpleString(maxFields: Int): String = {
val tableIdent = s"${catalog.name}.${ident.quoted}"
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
+ val order = sortOrder
+ .map { case (term, direction, nullOrder) =>
+ s"$term $direction $nullOrder"
+ }
+ .mkString(", ")
s"SetWriteDistributionAndOrdering $tableIdent $distributionMode $order"
}
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
index ecd3e0d3c9..07ac4aeda8 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -28,7 +27,8 @@ import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
- extends V2CommandExec with LeafExecNode {
+ extends V2CommandExec
+ with LeafExecNode {
override protected def run(): Seq[InternalRow] = {
val builder = new StringBuilder
@@ -43,7 +43,9 @@ case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
private def showColumns(view: View, builder: StringBuilder): Unit = {
val columns = concatByMultiLines(
- view.schema().fields
+ view
+ .schema()
+ .fields
.map(x => s"${x.name}${x.getComment().map(c => s" COMMENT '$c'").getOrElse("")}"))
builder ++= columns
}
@@ -54,14 +56,11 @@ case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
.foreach(builder.append)
}
- private def showProperties(
- view: View,
- builder: StringBuilder): Unit = {
+ private def showProperties(view: View, builder: StringBuilder): Unit = {
val showProps = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
if (showProps.nonEmpty) {
- val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map {
- case (key, value) =>
- s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
+ val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map { case (key, value) =>
+ s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
}
builder ++= "TBLPROPERTIES "
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
index 553b808330..ace43eb6c0 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -26,25 +25,22 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
-case class ShowV2ViewPropertiesExec(
- output: Seq[Attribute],
- view: View,
- propertyKey: Option[String]) extends V2CommandExec with LeafExecNode {
+case class ShowV2ViewPropertiesExec(output: Seq[Attribute], view: View, propertyKey: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
override protected def run(): Seq[InternalRow] = {
propertyKey match {
case Some(p) =>
- val propValue = properties.getOrElse(p,
- s"View ${view.name()} does not have property: $p")
+ val propValue = properties.getOrElse(p, s"View ${view.name()} does not have property: $p")
Seq(toCatalystRow(p, propValue))
case None =>
- properties.map {
- case (k, v) => toCatalystRow(k, v)
+ properties.map { case (k, v) =>
+ toCatalystRow(k, v)
}.toSeq
}
}
-
private def properties = {
view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
}
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
index 3aa85c3db5..83a1f84aec 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -27,10 +26,12 @@ import org.apache.spark.sql.execution.LeafExecNode
import scala.collection.mutable.ArrayBuffer
case class ShowV2ViewsExec(
- output: Seq[Attribute],
- catalog: ViewCatalog,
- namespace: Seq[String],
- pattern: Option[String]) extends V2CommandExec with LeafExecNode {
+ output: Seq[Attribute],
+ catalog: ViewCatalog,
+ namespace: Seq[String],
+ pattern: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -40,7 +41,8 @@ case class ShowV2ViewsExec(
// handle GLOBAL VIEWS
val globalTemp = session.sessionState.catalog.globalTempViewManager.database
if (namespace.nonEmpty && globalTemp == namespace.head) {
- pattern.map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
+ pattern
+ .map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
.getOrElse(session.sessionState.catalog.globalTempViewManager.listViewNames("*"))
.map(name => rows += toCatalystRow(globalTemp, name, true))
} else {
@@ -53,7 +55,8 @@ case class ShowV2ViewsExec(
}
// include TEMP VIEWS
- pattern.map(p => session.sessionState.catalog.listLocalTempViews(p))
+ pattern
+ .map(p => session.sessionState.catalog.listLocalTempViews(p))
.getOrElse(session.sessionState.catalog.listLocalTempViews("*"))
.map(v => rows += toCatalystRow(v.database.toArray.quoted, v.table, true))
diff --git a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
index 5acaa6800e..5d5990c23b 100644
--- a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
+++ b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.rdd.PartitionCoalescer
@@ -26,10 +25,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
// this node doesn't extend RepartitionOperation on purpose to keep this logic isolated
// and ignore it in optimizer rules such as CollapseRepartition
-case class OrderAwareCoalesce(
- numPartitions: Int,
- coalescer: PartitionCoalescer,
- child: LogicalPlan) extends OrderPreservingUnaryNode {
+case class OrderAwareCoalesce(numPartitions: Int, coalescer: PartitionCoalescer, child: LogicalPlan)
+ extends OrderPreservingUnaryNode {
override def output: Seq[Attribute] = child.output
diff --git a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
index 7b599eb3da..3ce3d3e82a 100644
--- a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
+++ b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.iceberg.DistributionMode
@@ -29,16 +28,19 @@ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits
case class SetWriteDistributionAndOrdering(
table: Seq[String],
distributionMode: Option[DistributionMode],
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafCommand {
+ sortOrder: Seq[(Term, SortDirection, NullOrder)])
+ extends LeafCommand {
import CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
override def simpleString(maxFields: Int): String = {
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
+ val order = sortOrder
+ .map { case (term, direction, nullOrder) =>
+ s"$term $direction $nullOrder"
+ }
+ .mkString(", ")
s"SetWriteDistributionAndOrdering ${table.quoted} $distributionMode $order"
}
}
diff --git a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
index bf19ef8a21..71b6107d04 100644
--- a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
+++ b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.iceberg.NullOrder
@@ -27,7 +26,9 @@ import org.apache.iceberg.expressions.Term
class SortOrderParserUtil {
- def collectSortOrder(tableSchema:Schema, sortOrder: Seq[(Term, SortDirection, NullOrder)]): SortOrder = {
+ def collectSortOrder(
+ tableSchema: Schema,
+ sortOrder: Seq[(Term, SortDirection, NullOrder)]): SortOrder = {
val orderBuilder = SortOrder.builderFor(tableSchema)
sortOrder.foreach {
case (term, SortDirection.ASC, nullOrder) =>
diff --git a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
index aa9e9c5533..f46127f818 100644
--- a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
+++ b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.utils
import org.apache.iceberg.spark.source.SparkTable
diff --git a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
index 2ef9955052..83dec98c0d 100644
--- a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
+++ b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution
import org.apache.spark.rdd.PartitionCoalescer
@@ -31,7 +30,8 @@ import org.apache.spark.sql.catalyst.plans.physical.UnknownPartitioning
case class OrderAwareCoalesceExec(
numPartitions: Int,
coalescer: PartitionCoalescer,
- child: SparkPlan) extends UnaryExecNode {
+ child: SparkPlan)
+ extends UnaryExecNode {
override def output: Seq[Attribute] = child.output
diff --git a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
index 95a0315e71..c50584be82 100644
--- a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
+++ b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources
import org.apache.iceberg.spark.SparkV2Filters
@@ -32,7 +31,8 @@ import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Strategy
object SparkExpressionConverter {
- def convertToIcebergExpression(sparkExpression: Expression): org.apache.iceberg.expressions.Expression = {
+ def convertToIcebergExpression(
+ sparkExpression: Expression): org.apache.iceberg.expressions.Expression = {
// Currently, it is a double conversion as we are converting Spark expression to Spark predicate
// and then converting Spark predicate to Iceberg expression.
// But these two conversions already exist and well tested. So, we are going with this approach.
@@ -40,26 +40,33 @@ object SparkExpressionConverter {
case Some(filter) =>
val converted = SparkV2Filters.convert(filter)
if (converted == null) {
- throw new IllegalArgumentException(s"Cannot convert Spark filter: $filter to Iceberg expression")
+ throw new IllegalArgumentException(
+ s"Cannot convert Spark filter: $filter to Iceberg expression")
}
converted
case _ =>
- throw new IllegalArgumentException(s"Cannot translate Spark expression: $sparkExpression to data source filter")
+ throw new IllegalArgumentException(
+ s"Cannot translate Spark expression: $sparkExpression to data source filter")
}
}
@throws[AnalysisException]
- def collectResolvedSparkExpression(session: SparkSession, tableName: String, where: String): Expression = {
+ def collectResolvedSparkExpression(
+ session: SparkSession,
+ tableName: String,
+ where: String): Expression = {
val tableAttrs = session.table(tableName).queryExecution.analyzed.output
val unresolvedExpression = session.sessionState.sqlParser.parseExpression(where)
val filter = Filter(unresolvedExpression, DummyRelation(tableAttrs))
val optimizedLogicalPlan = session.sessionState.executePlan(filter).optimizedPlan
- optimizedLogicalPlan.collectFirst {
- case filter: Filter => filter.condition
- case _: DummyRelation => Literal.TrueLiteral
- case _: LocalRelation => Literal.FalseLiteral
- }.getOrElse(throw new AnalysisException("Failed to find filter expression"))
+ optimizedLogicalPlan
+ .collectFirst {
+ case filter: Filter => filter.condition
+ case _: DummyRelation => Literal.TrueLiteral
+ case _: LocalRelation => Literal.FalseLiteral
+ }
+ .getOrElse(throw new AnalysisException("Failed to find filter expression"))
}
case class DummyRelation(output: Seq[Attribute]) extends LeafNode
diff --git a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
index cca16960f4..10e25311df 100644
--- a/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
+++ b/spark/v3.5/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.stats
import java.nio.ByteBuffer
@@ -46,12 +45,13 @@ import org.apache.spark.unsafe.types.UTF8String
* which should be deserialized to Compact sketch before using.
*
* See [[https://iceberg.apache.org/puffin-spec/]] for more information.
- *
*/
case class ThetaSketchAgg(
- child: Expression,
- mutableAggBufferOffset: Int = 0,
- inputAggBufferOffset: Int = 0) extends TypedImperativeAggregate[Sketch] with UnaryLike[Expression] {
+ child: Expression,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends TypedImperativeAggregate[Sketch]
+ with UnaryLike[Expression] {
private lazy val icebergType = SparkSchemaUtil.convert(child.dataType)
@@ -102,7 +102,8 @@ case class ThetaSketchAgg(
CompactSketch.wrap(Memory.wrap(storageFormat))
}
- override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate = {
+ override def withNewMutableAggBufferOffset(
+ newMutableAggBufferOffset: Int): ImperativeAggregate = {
copy(mutableAggBufferOffset = newMutableAggBufferOffset)
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
index 2dd4b08b3c..c4de35010c 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.iceberg.spark.extensions
import org.apache.spark.sql.SparkSessionExtensions
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
index 12c1fab36b..343fbe7a91 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.AnalysisException
@@ -38,12 +37,25 @@ object CheckViews extends (LogicalPlan => Unit) {
override def apply(plan: LogicalPlan): Unit = {
plan foreach {
- case CreateIcebergView(resolvedIdent@ResolvedIdentifier(_: ViewCatalog, _), _, query, columnAliases, _,
- _, _, _, _, replace, _) =>
+ case CreateIcebergView(
+ resolvedIdent @ ResolvedIdentifier(_: ViewCatalog, _),
+ _,
+ query,
+ columnAliases,
+ _,
+ _,
+ _,
+ _,
+ _,
+ replace,
+ _) =>
verifyColumnCount(resolvedIdent, columnAliases, query)
- SchemaUtils.checkColumnNameDuplication(query.schema.fieldNames.toIndexedSeq, SQLConf.get.resolver)
+ SchemaUtils.checkColumnNameDuplication(
+ query.schema.fieldNames.toIndexedSeq,
+ SQLConf.get.resolver)
if (replace) {
- val viewIdent: Seq[String] = resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
+ val viewIdent: Seq[String] =
+ resolvedIdent.catalog.name() +: resolvedIdent.identifier.asMultipartIdentifier
checkCyclicViewReference(viewIdent, query, Seq(viewIdent))
}
@@ -55,7 +67,10 @@ object CheckViews extends (LogicalPlan => Unit) {
}
}
- private def verifyColumnCount(ident: ResolvedIdentifier, columns: Seq[String], query: LogicalPlan): Unit = {
+ private def verifyColumnCount(
+ ident: ResolvedIdentifier,
+ columns: Seq[String],
+ query: LogicalPlan): Unit = {
if (columns.nonEmpty) {
if (columns.length > query.output.length) {
throw new AnalysisException(
@@ -76,11 +91,11 @@ object CheckViews extends (LogicalPlan => Unit) {
}
private def checkCyclicViewReference(
- viewIdent: Seq[String],
- plan: LogicalPlan,
- cyclePath: Seq[Seq[String]]): Unit = {
+ viewIdent: Seq[String],
+ plan: LogicalPlan,
+ cyclePath: Seq[Seq[String]]): Unit = {
plan match {
- case sub@SubqueryAlias(_, Project(_, _)) =>
+ case sub @ SubqueryAlias(_, Project(_, _)) =>
val currentViewIdent: Seq[String] = sub.identifier.qualifier :+ sub.identifier.name
checkIfRecursiveView(viewIdent, currentViewIdent, cyclePath, sub.children)
case v1View: View =>
@@ -99,15 +114,17 @@ object CheckViews extends (LogicalPlan => Unit) {
}
private def checkIfRecursiveView(
- viewIdent: Seq[String],
- currentViewIdent: Seq[String],
- cyclePath: Seq[Seq[String]],
- children: Seq[LogicalPlan]
- ): Unit = {
+ viewIdent: Seq[String],
+ currentViewIdent: Seq[String],
+ cyclePath: Seq[Seq[String]],
+ children: Seq[LogicalPlan]): Unit = {
val newCyclePath = cyclePath :+ currentViewIdent
if (currentViewIdent == viewIdent) {
- throw new IcebergAnalysisException(String.format("Recursive cycle in view detected: %s (cycle: %s)",
- viewIdent.asIdentifier, newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
+ throw new IcebergAnalysisException(
+ String.format(
+ "Recursive cycle in view detected: %s (cycle: %s)",
+ viewIdent.asIdentifier,
+ newCyclePath.map(p => p.mkString(".")).mkString(" -> ")))
} else {
children.foreach { c =>
checkCyclicViewReference(viewIdent, c, newCyclePath)
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
index fe94625234..23d33352f1 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SparkSession
@@ -47,36 +46,52 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager
override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
- case u@UnresolvedRelation(nameParts, _, _)
- if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
+ case u @ UnresolvedRelation(nameParts, _, _)
+ if catalogManager.v1SessionCatalog.isTempView(nameParts) =>
u
- case u@UnresolvedRelation(parts@CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
+ case u @ UnresolvedRelation(parts @ CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
.map(createViewRelation(parts, _))
.getOrElse(u)
- case u@UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
- ViewUtil.loadView(catalog, ident)
+ case u @ UnresolvedTableOrView(CatalogAndIdentifier(catalog, ident), _, _) =>
+ ViewUtil
+ .loadView(catalog, ident)
.map(_ => ResolvedV2View(catalog.asViewCatalog, ident))
.getOrElse(u)
- case c@CreateIcebergView(ResolvedIdentifier(_, _), _, query, columnAliases, columnComments, _, _, _, _, _, _)
- if query.resolved && !c.rewritten =>
+ case c @ CreateIcebergView(
+ ResolvedIdentifier(_, _),
+ _,
+ query,
+ columnAliases,
+ columnComments,
+ _,
+ _,
+ _,
+ _,
+ _,
+ _) if query.resolved && !c.rewritten =>
val aliased = aliasColumns(query, columnAliases, columnComments)
- c.copy(query = aliased, queryColumnNames = query.schema.fieldNames.toIndexedSeq, rewritten = true)
+ c.copy(
+ query = aliased,
+ queryColumnNames = query.schema.fieldNames.toIndexedSeq,
+ rewritten = true)
}
private def aliasColumns(
- plan: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]]): LogicalPlan = {
+ plan: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]]): LogicalPlan = {
if (columnAliases.isEmpty || columnAliases.length != plan.output.length) {
plan
} else {
val projectList = plan.output.zipWithIndex.map { case (attr, pos) =>
if (columnComments.apply(pos).isDefined) {
- val meta = new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
+ val meta =
+ new MetadataBuilder().putString("comment", columnComments.apply(pos).get).build()
Alias(attr, columnAliases.apply(pos))(explicitMetadata = Some(meta))
} else {
Alias(attr, columnAliases.apply(pos))()
@@ -86,7 +101,6 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
}
-
private def createViewRelation(nameParts: Seq[String], view: View): LogicalPlan = {
val parsed = parseViewText(nameParts.quoted, view.query)
@@ -99,17 +113,15 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
// This is more strict because it doesn't allow resolution by field name.
val aliases = view.schema.fields.zipWithIndex.map { case (expected, pos) =>
val attr = GetColumnByOrdinal(pos, expected.dataType)
- Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata = Some(expected.metadata))
+ Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata =
+ Some(expected.metadata))
}.toIndexedSeq
SubqueryAlias(nameParts, Project(aliases, rewritten))
}
private def parseViewText(name: String, viewText: String): LogicalPlan = {
- val origin = Origin(
- objectType = Some("VIEW"),
- objectName = Some(name)
- )
+ val origin = Origin(objectType = Some("VIEW"), objectName = Some(name))
try {
CurrentOrigin.withOrigin(origin) {
@@ -122,8 +134,8 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
private def rewriteIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = {
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = {
// Substitute CTEs and Unresolved Ordinals within the view, then rewrite unresolved functions and relations
qualifyTableIdentifiers(
qualifyFunctionIdentifiers(
@@ -133,15 +145,15 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
}
private def qualifyFunctionIdentifiers(
- plan: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
- case u@UnresolvedFunction(Seq(name), _, _, _, _, _, _) =>
+ plan: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan = plan transformExpressions {
+ case u @ UnresolvedFunction(Seq(name), _, _, _, _, _, _) =>
if (!isBuiltinFunction(name)) {
u.copy(nameParts = catalogAndNamespace :+ name)
} else {
u
}
- case u@UnresolvedFunction(parts, _, _, _, _, _, _) if !isCatalog(parts.head) =>
+ case u @ UnresolvedFunction(parts, _, _, _, _, _, _) if !isCatalog(parts.head) =>
u.copy(nameParts = catalogAndNamespace.head +: parts)
}
@@ -149,17 +161,16 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look
* Qualify table identifiers with default catalog and namespace if necessary.
*/
private def qualifyTableIdentifiers(
- child: LogicalPlan,
- catalogAndNamespace: Seq[String]): LogicalPlan =
+ child: LogicalPlan,
+ catalogAndNamespace: Seq[String]): LogicalPlan =
child transform {
- case u@UnresolvedRelation(Seq(table), _, _) =>
+ case u @ UnresolvedRelation(Seq(table), _, _) =>
u.copy(multipartIdentifier = catalogAndNamespace :+ table)
- case u@UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
+ case u @ UnresolvedRelation(parts, _, _) if !isCatalog(parts.head) =>
u.copy(multipartIdentifier = catalogAndNamespace.head +: parts)
case other =>
- other.transformExpressions {
- case subquery: SubqueryExpression =>
- subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
+ other.transformExpressions { case subquery: SubqueryExpression =>
+ subquery.withNewPlan(qualifyTableIdentifiers(subquery.plan, catalogAndNamespace))
}
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
index 1eeb50df8d..ac0f75c422 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SparkSession
@@ -51,11 +50,21 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case DropView(ResolvedIdent(resolved), ifExists) =>
DropIcebergView(resolved, ifExists)
- case CreateView(ResolvedIdent(resolved), userSpecifiedColumns, comment, _, properties,
- Some(queryText), query, allowExisting, replace, _) =>
+ case CreateView(
+ ResolvedIdent(resolved),
+ userSpecifiedColumns,
+ comment,
+ _,
+ properties,
+ Some(queryText),
+ query,
+ allowExisting,
+ replace,
+ _) =>
val q = CTESubstitution.apply(query)
verifyTemporaryObjectsDontExist(resolved, q)
- CreateIcebergView(child = resolved,
+ CreateIcebergView(
+ child = resolved,
queryText = queryText,
query = q,
columnAliases = userSpecifiedColumns.map(_._1),
@@ -67,20 +76,25 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case view @ ShowViews(CurrentNamespace, pattern, output) =>
if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) {
- ShowIcebergViews(ResolvedNamespace(catalogManager.currentCatalog, catalogManager.currentNamespace.toIndexedSeq),
- pattern, output)
+ ShowIcebergViews(
+ ResolvedNamespace(
+ catalogManager.currentCatalog,
+ catalogManager.currentNamespace.toIndexedSeq),
+ pattern,
+ output)
} else {
view
}
case ShowViews(UnresolvedNamespace(CatalogAndNamespace(catalog, ns), _), pattern, output)
- if ViewUtil.isViewCatalog(catalog) =>
+ if ViewUtil.isViewCatalog(catalog) =>
ShowIcebergViews(ResolvedNamespace(catalog, ns), pattern, output)
// needs to be done here instead of in ResolveViews, so that a V2 view can be resolved before the Analyzer
// tries to resolve it, which would result in an error, saying that V2 views aren't supported
- case u@UnresolvedView(ResolvedView(resolved), _, _, _) =>
- ViewUtil.loadView(resolved.catalog, resolved.identifier)
+ case u @ UnresolvedView(ResolvedView(resolved), _, _, _) =>
+ ViewUtil
+ .loadView(resolved.catalog, resolved.identifier)
.map(_ => ResolvedV2View(resolved.catalog.asViewCatalog, resolved.identifier))
.getOrElse(u)
}
@@ -101,7 +115,8 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case UnresolvedIdentifier(nameParts, true) if isTempView(nameParts) =>
None
- case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _) if ViewUtil.isViewCatalog(catalog) =>
+ case UnresolvedIdentifier(CatalogAndIdentifier(catalog, ident), _)
+ if ViewUtil.isViewCatalog(catalog) =>
Some(ResolvedIdentifier(catalog, ident))
case _ =>
@@ -113,11 +128,14 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
* Permanent views are not allowed to reference temp objects
*/
private def verifyTemporaryObjectsDontExist(
- identifier: ResolvedIdentifier,
- child: LogicalPlan): Unit = {
+ identifier: ResolvedIdentifier,
+ child: LogicalPlan): Unit = {
val tempViews = collectTemporaryViews(child)
if (tempViews.nonEmpty) {
- throw invalidRefToTempObject(identifier, tempViews.map(v => v.quoted).mkString("[", ", ", "]"), "view")
+ throw invalidRefToTempObject(
+ identifier,
+ tempViews.map(v => v.quoted).mkString("[", ", ", "]"),
+ "view")
}
val tempFunctions = collectTemporaryFunctions(child)
@@ -126,9 +144,17 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
}
}
- private def invalidRefToTempObject(ident: ResolvedIdentifier, tempObjectNames: String, tempObjectType: String) = {
- new IcebergAnalysisException(String.format("Cannot create view %s.%s that references temporary %s: %s",
- ident.catalog.name(), ident.identifier, tempObjectType, tempObjectNames))
+ private def invalidRefToTempObject(
+ ident: ResolvedIdentifier,
+ tempObjectNames: String,
+ tempObjectType: String) = {
+ new IcebergAnalysisException(
+ String.format(
+ "Cannot create view %s.%s that references temporary %s: %s",
+ ident.catalog.name(),
+ ident.identifier,
+ tempObjectType,
+ tempObjectNames))
}
/**
@@ -140,10 +166,11 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
case unresolved: UnresolvedRelation if isTempView(unresolved.multipartIdentifier) =>
Seq(unresolved.multipartIdentifier)
case view: View if view.isTempView => Seq(view.desc.identifier.nameParts)
- case plan => plan.expressions.flatMap(_.flatMap {
- case e: SubqueryExpression => collectTempViews(e.plan)
- case _ => Seq.empty
- })
+ case plan =>
+ plan.expressions.flatMap(_.flatMap {
+ case e: SubqueryExpression => collectTempViews(e.plan)
+ case _ => Seq.empty
+ })
}.distinct
}
@@ -156,7 +183,9 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi
None
case CatalogAndIdentifier(catalog, ident) if ViewUtil.isViewCatalog(catalog) =>
- ViewUtil.loadView(catalog, ident).flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
+ ViewUtil
+ .loadView(catalog, ident)
+ .flatMap(_ => Some(ResolvedV2View(catalog.asViewCatalog, ident)))
case _ =>
None
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
index d46f10b7f5..a7188837c5 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.connector.catalog.CatalogPlugin
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
index 5f78a0b279..bdec4aae88 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala
@@ -48,7 +48,7 @@ import org.apache.spark.sql.types.StructType
object ReplaceStaticInvoke extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan =
- plan.transformWithPruning (_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
+ plan.transformWithPruning(_.containsAnyPattern(COMMAND, FILTER, JOIN)) {
case replace @ ReplaceData(_, cond, _, _, _, _, _) =>
replaceStaticInvoke(replace, cond, newCond => replace.copy(condition = newCond))
@@ -57,7 +57,7 @@ object ReplaceStaticInvoke extends Rule[LogicalPlan] {
case filter @ Filter(cond, _) =>
replaceStaticInvoke(filter, cond, newCond => filter.copy(condition = newCond))
- }
+ }
private def replaceStaticInvoke[T <: LogicalPlan](
node: T,
@@ -90,16 +90,17 @@ object ReplaceStaticInvoke extends Rule[LogicalPlan] {
return invoke
}
- val inputType = StructType(invoke.arguments.zipWithIndex.map {
- case (exp, pos) => StructField(s"_$pos", exp.dataType, exp.nullable)
+ val inputType = StructType(invoke.arguments.zipWithIndex.map { case (exp, pos) =>
+ StructField(s"_$pos", exp.dataType, exp.nullable)
})
- val bound = try {
- unbound.bind(inputType)
- } catch {
- case _: Exception =>
- return invoke
- }
+ val bound =
+ try {
+ unbound.bind(inputType)
+ } catch {
+ case _: Exception =>
+ return invoke
+ }
if (bound.inputTypes().length != invoke.arguments.length) {
return invoke
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
index b8274983e5..25e056ee2d 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.parser.extensions
import java.util.Locale
@@ -45,7 +44,9 @@ import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.types.StructType
import scala.jdk.CollectionConverters._
-class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserInterface with ExtendedParser {
+class IcebergSparkSqlExtensionsParser(delegate: ParserInterface)
+ extends ParserInterface
+ with ExtendedParser {
import IcebergSparkSqlExtensionsParser._
@@ -108,7 +109,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
}.asJava
}
- override def parseRoutineParam(sqlText: String): StructType = throw new UnsupportedOperationException()
+ override def parseRoutineParam(sqlText: String): StructType =
+ throw new UnsupportedOperationException()
/**
* Parse a string to a LogicalPlan.
@@ -116,14 +118,17 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
override def parsePlan(sqlText: String): LogicalPlan = {
val sqlTextAfterSubstitution = substitutor.substitute(sqlText)
if (isIcebergCommand(sqlTextAfterSubstitution)) {
- parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }.asInstanceOf[LogicalPlan]
+ parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) }
+ .asInstanceOf[LogicalPlan]
} else {
RewriteViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText))
}
}
private def isIcebergCommand(sqlText: String): Boolean = {
- val normalized = sqlText.toLowerCase(Locale.ROOT).trim()
+ val normalized = sqlText
+ .toLowerCase(Locale.ROOT)
+ .trim()
// Strip simple SQL comments that terminate a line, e.g. comments starting with `--` .
.replaceAll("--.*?\\n", " ")
// Strip newlines.
@@ -135,30 +140,30 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
.replaceAll("`", "")
.trim()
- normalized.startsWith("alter table") && (
- normalized.contains("add partition field") ||
- normalized.contains("drop partition field") ||
- normalized.contains("replace partition field") ||
- normalized.contains("write ordered by") ||
- normalized.contains("write locally ordered by") ||
- normalized.contains("write distributed by") ||
- normalized.contains("write unordered") ||
- normalized.contains("set identifier fields") ||
- normalized.contains("drop identifier fields") ||
- isSnapshotRefDdl(normalized))
+ normalized.startsWith("alter table") && (normalized.contains("add partition field") ||
+ normalized.contains("drop partition field") ||
+ normalized.contains("replace partition field") ||
+ normalized.contains("write ordered by") ||
+ normalized.contains("write locally ordered by") ||
+ normalized.contains("write distributed by") ||
+ normalized.contains("write unordered") ||
+ normalized.contains("set identifier fields") ||
+ normalized.contains("drop identifier fields") ||
+ isSnapshotRefDdl(normalized))
}
private def isSnapshotRefDdl(normalized: String): Boolean = {
normalized.contains("create branch") ||
- normalized.contains("replace branch") ||
- normalized.contains("create tag") ||
- normalized.contains("replace tag") ||
- normalized.contains("drop branch") ||
- normalized.contains("drop tag")
+ normalized.contains("replace branch") ||
+ normalized.contains("create tag") ||
+ normalized.contains("replace tag") ||
+ normalized.contains("drop branch") ||
+ normalized.contains("drop tag")
}
protected def parse[T](command: String)(toResult: IcebergSqlExtensionsParser => T): T = {
- val lexer = new IcebergSqlExtensionsLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
+ val lexer = new IcebergSqlExtensionsLexer(
+ new UpperCaseCharStream(CharStreams.fromString(command)))
lexer.removeErrorListeners()
lexer.addErrorListener(IcebergParseErrorListener)
@@ -176,8 +181,7 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
parser.setErrorHandler(new BailErrorStrategy)
parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
toResult(parser)
- }
- catch {
+ } catch {
case _: ParseCancellationException =>
// if we fail, parse with LL mode with DefaultErrorStrategy
tokenStream.seek(0) // rewind input stream
@@ -188,8 +192,7 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
parser.getInterpreter.setPredictionMode(PredictionMode.LL)
toResult(parser)
}
- }
- catch {
+ } catch {
case e: IcebergParseException if e.command.isDefined =>
throw e
case e: IcebergParseException =>
@@ -207,7 +210,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
object IcebergSparkSqlExtensionsParser {
private val substitutorCtor: DynConstructors.Ctor[VariableSubstitution] =
- DynConstructors.builder()
+ DynConstructors
+ .builder()
.impl(classOf[VariableSubstitution])
.impl(classOf[VariableSubstitution], classOf[SQLConf])
.build()
@@ -253,9 +257,7 @@ case object IcebergSqlExtensionsPostProcessor extends IcebergSqlExtensionsBaseLi
replaceTokenByIdentifier(ctx, 0)(identity)
}
- private def replaceTokenByIdentifier(
- ctx: ParserRuleContext,
- stripMargins: Int)(
+ private def replaceTokenByIdentifier(ctx: ParserRuleContext, stripMargins: Int)(
f: CommonToken => CommonToken = identity): Unit = {
val parent = ctx.getParent
parent.removeLastChild()
@@ -302,10 +304,12 @@ class IcebergParseException(
val command: Option[String],
message: String,
val start: Origin,
- val stop: Origin) extends AnalysisException(message, start.line, start.startPosition) {
+ val stop: Origin)
+ extends AnalysisException(message, start.line, start.startPosition) {
def this(message: String, ctx: ParserRuleContext) = {
- this(Option(IcebergParserUtils.command(ctx)),
+ this(
+ Option(IcebergParserUtils.command(ctx)),
message,
IcebergParserUtils.position(ctx.getStart),
IcebergParserUtils.position(ctx.getStop))
@@ -315,8 +319,7 @@ class IcebergParseException(
val builder = new StringBuilder
builder ++= "\n" ++= message
start match {
- case Origin(Some(l), Some(p), Some(_), Some(_), Some(_), Some(_),
- Some(_), _, _) =>
+ case Origin(Some(l), Some(p), Some(_), Some(_), Some(_), Some(_), Some(_), _, _) =>
builder ++= s"(line $l, pos $p)\n"
command.foreach { cmd =>
val (above, below) = cmd.split("\n").splitAt(l)
@@ -336,4 +339,4 @@ class IcebergParseException(
def withCommand(cmd: String): IcebergParseException = {
new IcebergParseException(Option(cmd), message, start, stop)
}
-}
\ No newline at end of file
+}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
index 326bcf3895..724101cfe1 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.parser.extensions
import java.util.Locale
@@ -59,60 +58,69 @@ import org.apache.spark.sql.connector.expressions.LiteralValue
import org.apache.spark.sql.connector.expressions.Transform
import scala.jdk.CollectionConverters._
-class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
+class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface)
+ extends IcebergSqlExtensionsBaseVisitor[AnyRef] {
- private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] = list.asScala
+ private def toBuffer[T](list: java.util.List[T]): scala.collection.mutable.Buffer[T] =
+ list.asScala
private def toSeq[T](list: java.util.List[T]): Seq[T] = toBuffer(list).toSeq
/**
* Create an ADD PARTITION FIELD logical command.
*/
- override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField = withOrigin(ctx) {
- AddPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform),
- Option(ctx.name).map(_.getText))
- }
+ override def visitAddPartitionField(ctx: AddPartitionFieldContext): AddPartitionField =
+ withOrigin(ctx) {
+ AddPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform),
+ Option(ctx.name).map(_.getText))
+ }
/**
* Create a DROP PARTITION FIELD logical command.
*/
- override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField = withOrigin(ctx) {
- DropPartitionField(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- typedVisit[Transform](ctx.transform))
- }
+ override def visitDropPartitionField(ctx: DropPartitionFieldContext): DropPartitionField =
+ withOrigin(ctx) {
+ DropPartitionField(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ typedVisit[Transform](ctx.transform))
+ }
/**
* Create a CREATE OR REPLACE BRANCH logical command.
*/
- override def visitCreateOrReplaceBranch(ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
+ override def visitCreateOrReplaceBranch(
+ ctx: CreateOrReplaceBranchContext): CreateOrReplaceBranch = withOrigin(ctx) {
val createOrReplaceBranchClause = ctx.createReplaceBranchClause()
val branchName = createOrReplaceBranchClause.identifier()
val branchOptionsContext = Option(createOrReplaceBranchClause.branchOptions())
- val snapshotId = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotId()))
+ val snapshotId = branchOptionsContext
+ .flatMap(branchOptions => Option(branchOptions.snapshotId()))
.map(_.getText.toLong)
- val snapshotRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
- val minSnapshotsToKeep = snapshotRetention.flatMap(retention => Option(retention.minSnapshotsToKeep()))
+ val snapshotRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.snapshotRetention()))
+ val minSnapshotsToKeep = snapshotRetention
+ .flatMap(retention => Option(retention.minSnapshotsToKeep()))
.map(minSnapshots => minSnapshots.number().getText.toLong)
val maxSnapshotAgeMs = snapshotRetention
.flatMap(retention => Option(retention.maxSnapshotAge()))
- .map(retention => TimeUnit.valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
- .toMillis(retention.number().getText.toLong))
- val branchRetention = branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
+ .map(retention =>
+ TimeUnit
+ .valueOf(retention.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retention.number().getText.toLong))
+ val branchRetention =
+ branchOptionsContext.flatMap(branchOptions => Option(branchOptions.refRetain()))
val branchRefAgeMs = branchRetention.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
val create = createOrReplaceBranchClause.CREATE() != null
val replace = ctx.createReplaceBranchClause().REPLACE() != null
val ifNotExists = createOrReplaceBranchClause.EXISTS() != null
- val branchOptions = BranchOptions(
- snapshotId,
- minSnapshotsToKeep,
- maxSnapshotAgeMs,
- branchRefAgeMs
- )
+ val branchOptions =
+ BranchOptions(snapshotId, minSnapshotsToKeep, maxSnapshotAgeMs, branchRefAgeMs)
CreateOrReplaceBranch(
typedVisit[Seq[String]](ctx.multipartIdentifier),
@@ -126,52 +134,61 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an CREATE OR REPLACE TAG logical command.
*/
- override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag = withOrigin(ctx) {
- val createTagClause = ctx.createReplaceTagClause()
-
- val tagName = createTagClause.identifier().getText
-
- val tagOptionsContext = Option(createTagClause.tagOptions())
- val snapshotId = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.snapshotId()))
- .map(_.getText.toLong)
- val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
- val tagRefAgeMs = tagRetain.map(retain =>
- TimeUnit.valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH)).toMillis(retain.number().getText.toLong))
- val tagOptions = TagOptions(
- snapshotId,
- tagRefAgeMs
- )
-
- val create = createTagClause.CREATE() != null
- val replace = createTagClause.REPLACE() != null
- val ifNotExists = createTagClause.EXISTS() != null
-
- CreateOrReplaceTag(typedVisit[Seq[String]](ctx.multipartIdentifier),
- tagName,
- tagOptions,
- create,
- replace,
- ifNotExists)
- }
+ override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTag =
+ withOrigin(ctx) {
+ val createTagClause = ctx.createReplaceTagClause()
+
+ val tagName = createTagClause.identifier().getText
+
+ val tagOptionsContext = Option(createTagClause.tagOptions())
+ val snapshotId = tagOptionsContext
+ .flatMap(tagOptions => Option(tagOptions.snapshotId()))
+ .map(_.getText.toLong)
+ val tagRetain = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.refRetain()))
+ val tagRefAgeMs = tagRetain.map(retain =>
+ TimeUnit
+ .valueOf(retain.timeUnit().getText.toUpperCase(Locale.ENGLISH))
+ .toMillis(retain.number().getText.toLong))
+ val tagOptions = TagOptions(snapshotId, tagRefAgeMs)
+
+ val create = createTagClause.CREATE() != null
+ val replace = createTagClause.REPLACE() != null
+ val ifNotExists = createTagClause.EXISTS() != null
+
+ CreateOrReplaceTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ tagName,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists)
+ }
/**
* Create an DROP BRANCH logical command.
*/
override def visitDropBranch(ctx: DropBranchContext): DropBranch = withOrigin(ctx) {
- DropBranch(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
+ DropBranch(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
}
/**
* Create an DROP TAG logical command.
*/
override def visitDropTag(ctx: DropTagContext): DropTag = withOrigin(ctx) {
- DropTag(typedVisit[Seq[String]](ctx.multipartIdentifier), ctx.identifier().getText, ctx.EXISTS() != null)
+ DropTag(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ ctx.identifier().getText,
+ ctx.EXISTS() != null)
}
/**
* Create an REPLACE PARTITION FIELD logical command.
*/
- override def visitReplacePartitionField(ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
+ override def visitReplacePartitionField(
+ ctx: ReplacePartitionFieldContext): ReplacePartitionField = withOrigin(ctx) {
ReplacePartitionField(
typedVisit[Seq[String]](ctx.multipartIdentifier),
typedVisit[Transform](ctx.transform(0)),
@@ -182,20 +199,22 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create an SET IDENTIFIER FIELDS logical command.
*/
- override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields = withOrigin(ctx) {
- SetIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
+ override def visitSetIdentifierFields(ctx: SetIdentifierFieldsContext): SetIdentifierFields =
+ withOrigin(ctx) {
+ SetIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
/**
* Create an DROP IDENTIFIER FIELDS logical command.
*/
- override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields = withOrigin(ctx) {
- DropIdentifierFields(
- typedVisit[Seq[String]](ctx.multipartIdentifier),
- toSeq(ctx.fieldList.fields).map(_.getText))
- }
+ override def visitDropIdentifierFields(ctx: DropIdentifierFieldsContext): DropIdentifierFields =
+ withOrigin(ctx) {
+ DropIdentifierFields(
+ typedVisit[Seq[String]](ctx.multipartIdentifier),
+ toSeq(ctx.fieldList.fields).map(_.getText))
+ }
/**
* Create a [[SetWriteDistributionAndOrdering]] for changing the write distribution and ordering.
@@ -253,12 +272,15 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
*/
override def visitOrderField(ctx: OrderFieldContext): (Term, SortDirection, NullOrder) = {
val term = Spark3Util.toIcebergTerm(typedVisit[Transform](ctx.transform))
- val direction = Option(ctx.ASC).map(_ => SortDirection.ASC)
- .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
- .getOrElse(SortDirection.ASC)
- val nullOrder = Option(ctx.FIRST).map(_ => NullOrder.NULLS_FIRST)
- .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
- .getOrElse(if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
+ val direction = Option(ctx.ASC)
+ .map(_ => SortDirection.ASC)
+ .orElse(Option(ctx.DESC).map(_ => SortDirection.DESC))
+ .getOrElse(SortDirection.ASC)
+ val nullOrder = Option(ctx.FIRST)
+ .map(_ => NullOrder.NULLS_FIRST)
+ .orElse(Option(ctx.LAST).map(_ => NullOrder.NULLS_LAST))
+ .getOrElse(
+ if (direction == SortDirection.ASC) NullOrder.NULLS_FIRST else NullOrder.NULLS_LAST)
(term, direction, nullOrder)
}
@@ -280,27 +302,31 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
/**
* Create a transform argument from a column reference or a constant.
*/
- override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression = withOrigin(ctx) {
- val reference = Option(ctx.multipartIdentifier())
+ override def visitTransformArgument(ctx: TransformArgumentContext): expressions.Expression =
+ withOrigin(ctx) {
+ val reference = Option(ctx.multipartIdentifier())
.map(typedVisit[Seq[String]])
.map(FieldReference(_))
- val literal = Option(ctx.constant)
+ val literal = Option(ctx.constant)
.map(visitConstant)
.map(lit => LiteralValue(lit.value, lit.dataType))
- reference.orElse(literal)
+ reference
+ .orElse(literal)
.getOrElse(throw new IcebergParseException(s"Invalid transform argument", ctx))
- }
+ }
/**
* Return a multi-part identifier as Seq[String].
*/
- override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) {
- toSeq(ctx.parts).map(_.getText)
- }
+ override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] =
+ withOrigin(ctx) {
+ toSeq(ctx.parts).map(_.getText)
+ }
- override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] = withOrigin(ctx) {
- toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
- }
+ override def visitSingleOrder(ctx: SingleOrderContext): Seq[(Term, SortDirection, NullOrder)] =
+ withOrigin(ctx) {
+ toSeq(ctx.order.fields).map(typedVisit[(Term, SortDirection, NullOrder)])
+ }
override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) {
visit(ctx.statement).asInstanceOf[LogicalPlan]
@@ -320,10 +346,12 @@ class IcebergSqlExtensionsAstBuilder(delegate: ParserInterface) extends IcebergS
}
private def reconstructSqlString(ctx: ParserRuleContext): String = {
- toBuffer(ctx.children).map {
- case c: ParserRuleContext => reconstructSqlString(c)
- case t: TerminalNode => t.getText
- }.mkString(" ")
+ toBuffer(ctx.children)
+ .map {
+ case c: ParserRuleContext => reconstructSqlString(c)
+ case t: TerminalNode => t.getText
+ }
+ .mkString(" ")
}
private def typedVisit[T](ctx: ParseTree): T = {
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
index e8b1b29411..0a830dbd4f 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala
@@ -16,13 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.expressions.Transform
-case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String]) extends LeafCommand {
+case class AddPartitionField(table: Seq[String], transform: Transform, name: Option[String])
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
index 4d7e0a086b..15b9083002 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala
@@ -16,8 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
-case class BranchOptions (snapshotId: Option[Long], numSnapshots: Option[Long],
- snapshotRetain: Option[Long], snapshotRefRetain: Option[Long])
+case class BranchOptions(
+ snapshotId: Option[Long],
+ numSnapshots: Option[Long],
+ snapshotRetain: Option[Long],
+ snapshotRefRetain: Option[Long])
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
index b7981a3c7a..6900f6e8cc 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceBranch.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -27,7 +26,8 @@ case class CreateOrReplaceBranch(
branchOptions: BranchOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
+ ifNotExists: Boolean)
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
index 6e7db84a90..957c68e7a5 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -27,7 +26,8 @@ case class CreateOrReplaceTag(
tagOptions: TagOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafCommand {
+ ifNotExists: Boolean)
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
index bee0b0fae6..ed4f1f512b 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropBranch.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
index 29dd686a0f..1a91806280 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala
@@ -16,14 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
-case class DropIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
+case class DropIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
index fb14513241..ec952c8c71 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
index 7e4b38e74d..da69ca0383 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
index 8c660c6f37..c2525369e7 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -26,13 +25,14 @@ case class ReplacePartitionField(
table: Seq[String],
transformFrom: Transform,
transformTo: Transform,
- name: Option[String]) extends LeafCommand {
+ name: Option[String])
+ extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
override def simpleString(maxFields: Int): String = {
s"ReplacePartitionField ${table.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
}
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
index 1e5622e9c3..8cd2c0ddad 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala
@@ -16,14 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.Attribute
-case class SetIdentifierFields(
- table: Seq[String],
- fields: Seq[String]) extends LeafCommand {
+case class SetIdentifierFields(table: Seq[String], fields: Seq[String]) extends LeafCommand {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
index 85e3b95f4a..6afe1478d7 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
case class TagOptions(snapshotId: Option[Long], snapshotRefRetain: Option[Long])
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
index 9366d5efe1..c5dadae701 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
@@ -16,29 +16,30 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.plans.logical.BinaryCommand
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
case class CreateIcebergView(
- child: LogicalPlan,
- queryText: String,
- query: LogicalPlan,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String] = Seq.empty,
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean,
- rewritten: Boolean = false) extends BinaryCommand {
+ child: LogicalPlan,
+ queryText: String,
+ query: LogicalPlan,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String] = Seq.empty,
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean,
+ rewritten: Boolean = false)
+ extends BinaryCommand {
override def left: LogicalPlan = child
override def right: LogicalPlan = query
override protected def withNewChildrenInternal(
- newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+ newLeft: LogicalPlan,
+ newRight: LogicalPlan): LogicalPlan =
copy(child = newLeft, query = newRight)
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
index 275dba6fbf..092b6b33fb 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/DropIcebergView.scala
@@ -16,15 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
-case class DropIcebergView(
- child: LogicalPlan,
- ifExists: Boolean) extends UnaryCommand {
+case class DropIcebergView(child: LogicalPlan, ifExists: Boolean) extends UnaryCommand {
override protected def withNewChildInternal(newChild: LogicalPlan): DropIcebergView =
copy(child = newChild)
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
index b9c05ff006..4d384e8577 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ResolvedV2View.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.analysis.LeafNodeWithoutStats
@@ -24,8 +23,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-case class ResolvedV2View(
- catalog: ViewCatalog,
- identifier: Identifier) extends LeafNodeWithoutStats {
+case class ResolvedV2View(catalog: ViewCatalog, identifier: Identifier)
+ extends LeafNodeWithoutStats {
override def output: Seq[Attribute] = Nil
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
index b09c27acdc..cbfe23d94c 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
@@ -16,8 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-
package org.apache.spark.sql.catalyst.plans.logical.views
import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -26,9 +24,10 @@ import org.apache.spark.sql.catalyst.plans.logical.ShowViews
import org.apache.spark.sql.catalyst.plans.logical.UnaryCommand
case class ShowIcebergViews(
- namespace: LogicalPlan,
- pattern: Option[String],
- override val output: Seq[Attribute] = ShowViews.getOutputAttrs) extends UnaryCommand {
+ namespace: LogicalPlan,
+ pattern: Option[String],
+ override val output: Seq[Attribute] = ShowViews.getOutputAttrs)
+ extends UnaryCommand {
override def child: LogicalPlan = namespace
override protected def withNewChildInternal(newChild: LogicalPlan): ShowIcebergViews =
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
index 55f327f7e4..e28dcfb194 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -31,7 +30,8 @@ case class AddPartitionFieldExec(
catalog: TableCatalog,
ident: Identifier,
transform: Transform,
- name: Option[String]) extends LeafV2CommandExec {
+ name: Option[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -39,12 +39,14 @@ case class AddPartitionFieldExec(
override protected def run(): Seq[InternalRow] = {
catalog.loadTable(ident) match {
case iceberg: SparkTable =>
- iceberg.table.updateSpec()
- .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transform))
+ .commit()
case table =>
- throw new UnsupportedOperationException(s"Cannot add partition field to non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot add partition field to non-Iceberg table: $table")
}
Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
index b103d1ee2c..d6630e51ff 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,17 +24,17 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.connector.catalog.ViewChange
-
case class AlterV2ViewSetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- properties: Map[String, String]) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ properties: Map[String, String])
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
override protected def run(): Seq[InternalRow] = {
- val changes = properties.map {
- case (property, value) => ViewChange.setProperty(property, value)
+ val changes = properties.map { case (property, value) =>
+ ViewChange.setProperty(property, value)
}.toSeq
catalog.alterView(ident, changes: _*)
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
index d612173ca2..aa57842f58 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -26,12 +25,12 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.connector.catalog.ViewChange
-
case class AlterV2ViewUnsetPropertiesExec(
- catalog: ViewCatalog,
- ident: Identifier,
- propertyKeys: Seq[String],
- ifExists: Boolean) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ propertyKeys: Seq[String],
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
index 2be406e7f3..baf985f53a 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceBranchExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -28,13 +27,14 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
case class CreateOrReplaceBranchExec(
- catalog: TableCatalog,
- ident: Identifier,
- branch: String,
- branchOptions: BranchOptions,
- create: Boolean,
- replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
+ catalog: TableCatalog,
+ ident: Identifier,
+ branch: String,
+ branchOptions: BranchOptions,
+ create: Boolean,
+ replace: Boolean,
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -62,8 +62,10 @@ case class CreateOrReplaceBranchExec(
if (create && replace && !refExists) {
safeCreateBranch()
} else if (replace) {
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete replace branch operation on %s, main has no snapshot", ident)
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete replace branch operation on %s, main has no snapshot",
+ ident)
manageSnapshots.replaceBranch(branch, snapshotId)
} else {
if (refExists && ifNotExists) {
@@ -88,7 +90,8 @@ case class CreateOrReplaceBranchExec(
manageSnapshots.commit()
case table =>
- throw new UnsupportedOperationException(s"Cannot create or replace branch on non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot create or replace branch on non-Iceberg table: $table")
}
Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
index 372cd75486..e486892614 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -33,7 +32,8 @@ case class CreateOrReplaceTagExec(
tagOptions: TagOptions,
create: Boolean,
replace: Boolean,
- ifNotExists: Boolean) extends LeafV2CommandExec {
+ ifNotExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -47,8 +47,10 @@ case class CreateOrReplaceTagExec(
.map(java.lang.Long.valueOf)
.orNull
- Preconditions.checkArgument(snapshotId != null,
- "Cannot complete create or replace tag operation on %s, main has no snapshot", ident)
+ Preconditions.checkArgument(
+ snapshotId != null,
+ "Cannot complete create or replace tag operation on %s, main has no snapshot",
+ ident)
val manageSnapshot = iceberg.table.manageSnapshots()
val refExists = null != iceberg.table().refs().get(tag)
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
index 97077a5ecf..04f2b2af73 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.SupportsReplaceView
@@ -30,31 +29,33 @@ import org.apache.spark.sql.connector.catalog.ViewInfo
import org.apache.spark.sql.types.StructType
import scala.jdk.CollectionConverters._
-
case class CreateV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- queryText: String,
- viewSchema: StructType,
- columnAliases: Seq[String],
- columnComments: Seq[Option[String]],
- queryColumnNames: Seq[String],
- comment: Option[String],
- properties: Map[String, String],
- allowExisting: Boolean,
- replace: Boolean) extends LeafV2CommandExec {
+ catalog: ViewCatalog,
+ ident: Identifier,
+ queryText: String,
+ viewSchema: StructType,
+ columnAliases: Seq[String],
+ columnComments: Seq[Option[String]],
+ queryColumnNames: Seq[String],
+ comment: Option[String],
+ properties: Map[String, String],
+ allowExisting: Boolean,
+ replace: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
override protected def run(): Seq[InternalRow] = {
val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name
- val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
+ val currentCatalog =
+ if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null
val currentNamespace = session.sessionState.catalogManager.currentNamespace
val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION
val newProperties = properties ++
comment.map(ViewCatalog.PROP_COMMENT -> _) ++
- Map(ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
+ Map(
+ ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion,
ViewCatalog.PROP_ENGINE_VERSION -> engineVersion)
if (replace) {
@@ -88,10 +89,10 @@ case class CreateV2ViewExec(
}
private def replaceView(
- supportsReplaceView: SupportsReplaceView,
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
+ supportsReplaceView: SupportsReplaceView,
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
supportsReplaceView.replaceView(
ident,
queryText,
@@ -105,9 +106,9 @@ case class CreateV2ViewExec(
}
private def createView(
- currentCatalog: String,
- currentNamespace: Array[String],
- newProperties: Map[String, String]) = {
+ currentCatalog: String,
+ currentNamespace: Array[String],
+ newProperties: Map[String, String]) = {
val viewInfo: ViewInfo = new ViewInfo(
ident,
queryText,
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
index 38385d2979..106734d207 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -27,10 +26,9 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
-case class DescribeV2ViewExec(
- output: Seq[Attribute],
- view: View,
- isExtended: Boolean) extends V2CommandExec with LeafExecNode {
+case class DescribeV2ViewExec(output: Seq[Attribute], view: View, isExtended: Boolean)
+ extends V2CommandExec
+ with LeafExecNode {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -44,23 +42,22 @@ case class DescribeV2ViewExec(
private def describeSchema: Seq[InternalRow] =
view.schema().map { column =>
- toCatalystRow(
- column.name,
- column.dataType.simpleString,
- column.getComment().getOrElse(""))
+ toCatalystRow(column.name, column.dataType.simpleString, column.getComment().getOrElse(""))
}
private def emptyRow: InternalRow = toCatalystRow("", "", "")
private def describeExtended: Seq[InternalRow] = {
val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
- val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
+ val properties: Map[String, String] =
+ view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2).toIndexedSeq
- val viewProperties = properties.toSeq.sortBy(_._1).map {
- case (key, value) =>
+ val viewProperties = properties.toSeq
+ .sortBy(_._1)
+ .map { case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
- }.mkString("[", ", ", "]")
-
+ }
+ .mkString("[", ", ", "]")
// omitting view text here because it is shown as
// part of SHOW CREATE TABLE and can result in weird formatting in the DESCRIBE output
@@ -69,7 +66,10 @@ case class DescribeV2ViewExec(
toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
toCatalystRow("View Query Output Columns", outputColumns, "") ::
toCatalystRow("View Properties", viewProperties, "") ::
- toCatalystRow("Created By", view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""), "") ::
+ toCatalystRow(
+ "Created By",
+ view.properties.getOrDefault(ViewCatalog.PROP_CREATE_ENGINE_VERSION, ""),
+ "") ::
Nil
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
index ff8f182009..e7d9c7b70d 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropBranchExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -29,7 +28,8 @@ case class DropBranchExec(
catalog: TableCatalog,
ident: Identifier,
branch: String,
- ifExists: Boolean) extends LeafV2CommandExec {
+ ifExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
index dee778b474..87b18594d5 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.relocated.com.google.common.base.Preconditions
@@ -27,10 +26,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
-case class DropIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
+case class DropIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -42,18 +39,24 @@ case class DropIdentifierFieldsExec(
val identifierFieldNames = Sets.newHashSet(schema.identifierFieldNames)
for (name <- fields) {
- Preconditions.checkArgument(schema.findField(name) != null,
- "Cannot complete drop identifier fields operation: field %s not found", name)
- Preconditions.checkArgument(identifierFieldNames.contains(name),
- "Cannot complete drop identifier fields operation: %s is not an identifier field", name)
+ Preconditions.checkArgument(
+ schema.findField(name) != null,
+ "Cannot complete drop identifier fields operation: field %s not found",
+ name)
+ Preconditions.checkArgument(
+ identifierFieldNames.contains(name),
+ "Cannot complete drop identifier fields operation: %s is not an identifier field",
+ name)
identifierFieldNames.remove(name)
}
- iceberg.table.updateSchema()
+ iceberg.table
+ .updateSchema()
.setIdentifierFields(identifierFieldNames)
.commit();
case table =>
- throw new UnsupportedOperationException(s"Cannot drop identifier fields in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot drop identifier fields in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
index 9a153f0c00..db43263e0e 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -29,10 +28,8 @@ import org.apache.spark.sql.connector.expressions.FieldReference
import org.apache.spark.sql.connector.expressions.IdentityTransform
import org.apache.spark.sql.connector.expressions.Transform
-case class DropPartitionFieldExec(
- catalog: TableCatalog,
- ident: Identifier,
- transform: Transform) extends LeafV2CommandExec {
+case class DropPartitionFieldExec(catalog: TableCatalog, ident: Identifier, transform: Transform)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -42,20 +39,24 @@ case class DropPartitionFieldExec(
case iceberg: SparkTable =>
val schema = iceberg.table.schema
transform match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
// the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .commit()
case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transform))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transform))
+ .commit()
}
case table =>
- throw new UnsupportedOperationException(s"Cannot drop partition field in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot drop partition field in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
index 8df88765a9..79b4a15255 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -25,11 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
-case class DropTagExec(
- catalog: TableCatalog,
- ident: Identifier,
- tag: String,
- ifExists: Boolean) extends LeafV2CommandExec {
+case class DropTagExec(catalog: TableCatalog, ident: Identifier, tag: String, ifExists: Boolean)
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
index c35af1486f..6dd1188b78 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -25,11 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-case class DropV2ViewExec(
- catalog: ViewCatalog,
- ident: Identifier,
- ifExists: Boolean) extends LeafV2CommandExec {
+case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boolean)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
index 9958d50840..898e6f3d41 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -64,11 +63,28 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
AddPartitionFieldExec(catalog, ident, transform, name) :: Nil
case CreateOrReplaceBranch(
- IcebergCatalogAndIdentifier(catalog, ident), branch, branchOptions, create, replace, ifNotExists) =>
- CreateOrReplaceBranchExec(catalog, ident, branch, branchOptions, create, replace, ifNotExists) :: Nil
+ IcebergCatalogAndIdentifier(catalog, ident),
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) =>
+ CreateOrReplaceBranchExec(
+ catalog,
+ ident,
+ branch,
+ branchOptions,
+ create,
+ replace,
+ ifNotExists) :: Nil
case CreateOrReplaceTag(
- IcebergCatalogAndIdentifier(catalog, ident), tag, tagOptions, create, replace, ifNotExists) =>
+ IcebergCatalogAndIdentifier(catalog, ident),
+ tag,
+ tagOptions,
+ create,
+ replace,
+ ifNotExists) =>
CreateOrReplaceTagExec(catalog, ident, tag, tagOptions, create, replace, ifNotExists) :: Nil
case DropBranch(IcebergCatalogAndIdentifier(catalog, ident), branch, ifExists) =>
@@ -80,7 +96,11 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
case DropPartitionField(IcebergCatalogAndIdentifier(catalog, ident), transform) =>
DropPartitionFieldExec(catalog, ident, transform) :: Nil
- case ReplacePartitionField(IcebergCatalogAndIdentifier(catalog, ident), transformFrom, transformTo, name) =>
+ case ReplacePartitionField(
+ IcebergCatalogAndIdentifier(catalog, ident),
+ transformFrom,
+ transformTo,
+ name) =>
ReplacePartitionFieldExec(catalog, ident, transformFrom, transformTo, name) :: Nil
case SetIdentifierFields(IcebergCatalogAndIdentifier(catalog, ident), fields) =>
@@ -90,13 +110,15 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
DropIdentifierFieldsExec(catalog, ident, fields) :: Nil
case SetWriteDistributionAndOrdering(
- IcebergCatalogAndIdentifier(catalog, ident), distributionMode, ordering) =>
+ IcebergCatalogAndIdentifier(catalog, ident),
+ distributionMode,
+ ordering) =>
SetWriteDistributionAndOrderingExec(catalog, ident, distributionMode, ordering) :: Nil
case OrderAwareCoalesce(numPartitions, coalescer, child) =>
OrderAwareCoalesceExec(numPartitions, coalescer, planLater(child)) :: Nil
- case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView@true) =>
+ case RenameTable(ResolvedV2View(oldCatalog: ViewCatalog, oldIdent), newName, isView @ true) =>
val newIdent = Spark3Util.catalogAndIdentifier(spark, newName.toList.asJava)
if (oldCatalog.name != newIdent.catalog().name()) {
throw new IcebergAnalysisException(
@@ -107,8 +129,18 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi
case DropIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), ifExists) =>
DropV2ViewExec(viewCatalog, ident, ifExists) :: Nil
- case CreateIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), queryText, query,
- columnAliases, columnComments, queryColumnNames, comment, properties, allowExisting, replace, _) =>
+ case CreateIcebergView(
+ ResolvedIdentifier(viewCatalog: ViewCatalog, ident),
+ queryText,
+ query,
+ columnAliases,
+ columnComments,
+ queryColumnNames,
+ comment,
+ properties,
+ allowExisting,
+ replace,
+ _) =>
CreateV2ViewExec(
catalog = viewCatalog,
ident = ident,
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
index 61d362044c..5dada1cab0 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RenameV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -24,11 +23,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.ViewCatalog
-
-case class RenameV2ViewExec(
- catalog: ViewCatalog,
- oldIdent: Identifier,
- newIdent: Identifier) extends LeafV2CommandExec {
+case class RenameV2ViewExec(catalog: ViewCatalog, oldIdent: Identifier, newIdent: Identifier)
+ extends LeafV2CommandExec {
override lazy val output: Seq[Attribute] = Nil
@@ -38,7 +34,6 @@ case class RenameV2ViewExec(
Seq.empty
}
-
override def simpleString(maxFields: Int): String = {
s"RenameV2View ${oldIdent} to {newIdent}"
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
index fcae0a5def..00b998c49e 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.Spark3Util
@@ -34,7 +33,8 @@ case class ReplacePartitionFieldExec(
ident: Identifier,
transformFrom: Transform,
transformTo: Transform,
- name: Option[String]) extends LeafV2CommandExec {
+ name: Option[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -44,22 +44,26 @@ case class ReplacePartitionFieldExec(
case iceberg: SparkTable =>
val schema = iceberg.table.schema
transformFrom match {
- case IdentityTransform(FieldReference(parts)) if parts.size == 1 && schema.findField(parts.head) == null =>
+ case IdentityTransform(FieldReference(parts))
+ if parts.size == 1 && schema.findField(parts.head) == null =>
// the name is not present in the Iceberg schema, so it must be a partition field name, not a column name
- iceberg.table.updateSpec()
- .removeField(parts.head)
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(parts.head)
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
case _ =>
- iceberg.table.updateSpec()
- .removeField(Spark3Util.toIcebergTerm(transformFrom))
- .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
- .commit()
+ iceberg.table
+ .updateSpec()
+ .removeField(Spark3Util.toIcebergTerm(transformFrom))
+ .addField(name.orNull, Spark3Util.toIcebergTerm(transformTo))
+ .commit()
}
case table =>
- throw new UnsupportedOperationException(s"Cannot replace partition field in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot replace partition field in non-Iceberg table: $table")
}
Nil
@@ -67,6 +71,6 @@ case class ReplacePartitionFieldExec(
override def simpleString(maxFields: Int): String = {
s"ReplacePartitionField ${catalog.name}.${ident.quoted} ${transformFrom.describe} " +
- s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
+ s"with ${name.map(n => s"$n=").getOrElse("")}${transformTo.describe}"
}
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
index b50550ad38..50c53473ab 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.spark.source.SparkTable
@@ -26,10 +25,8 @@ import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.TableCatalog
import scala.jdk.CollectionConverters._
-case class SetIdentifierFieldsExec(
- catalog: TableCatalog,
- ident: Identifier,
- fields: Seq[String]) extends LeafV2CommandExec {
+case class SetIdentifierFieldsExec(catalog: TableCatalog, ident: Identifier, fields: Seq[String])
+ extends LeafV2CommandExec {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
@@ -37,11 +34,13 @@ case class SetIdentifierFieldsExec(
override protected def run(): Seq[InternalRow] = {
catalog.loadTable(ident) match {
case iceberg: SparkTable =>
- iceberg.table.updateSchema()
+ iceberg.table
+ .updateSchema()
.setIdentifierFields(fields.asJava)
.commit();
case table =>
- throw new UnsupportedOperationException(s"Cannot set identifier fields in non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot set identifier fields in non-Iceberg table: $table")
}
Nil
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
index c9004ddc5b..9a10949d5e 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.iceberg.DistributionMode
@@ -36,7 +35,8 @@ case class SetWriteDistributionAndOrderingExec(
catalog: TableCatalog,
ident: Identifier,
distributionMode: Option[DistributionMode],
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafV2CommandExec {
+ sortOrder: Seq[(Term, SortDirection, NullOrder)])
+ extends LeafV2CommandExec {
import CatalogV2Implicits._
@@ -57,7 +57,8 @@ case class SetWriteDistributionAndOrderingExec(
orderBuilder.commit()
distributionMode.foreach { mode =>
- txn.updateProperties()
+ txn
+ .updateProperties()
.set(WRITE_DISTRIBUTION_MODE, mode.modeName())
.commit()
}
@@ -65,7 +66,8 @@ case class SetWriteDistributionAndOrderingExec(
txn.commitTransaction()
case table =>
- throw new UnsupportedOperationException(s"Cannot set write order of non-Iceberg table: $table")
+ throw new UnsupportedOperationException(
+ s"Cannot set write order of non-Iceberg table: $table")
}
Nil
@@ -73,9 +75,11 @@ case class SetWriteDistributionAndOrderingExec(
override def simpleString(maxFields: Int): String = {
val tableIdent = s"${catalog.name}.${ident.quoted}"
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
+ val order = sortOrder
+ .map { case (term, direction, nullOrder) =>
+ s"$term $direction $nullOrder"
+ }
+ .mkString(", ")
s"SetWriteDistributionAndOrdering $tableIdent $distributionMode $order"
}
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
index ecd3e0d3c9..07ac4aeda8 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -28,7 +27,8 @@ import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
- extends V2CommandExec with LeafExecNode {
+ extends V2CommandExec
+ with LeafExecNode {
override protected def run(): Seq[InternalRow] = {
val builder = new StringBuilder
@@ -43,7 +43,9 @@ case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
private def showColumns(view: View, builder: StringBuilder): Unit = {
val columns = concatByMultiLines(
- view.schema().fields
+ view
+ .schema()
+ .fields
.map(x => s"${x.name}${x.getComment().map(c => s" COMMENT '$c'").getOrElse("")}"))
builder ++= columns
}
@@ -54,14 +56,11 @@ case class ShowCreateV2ViewExec(output: Seq[Attribute], view: View)
.foreach(builder.append)
}
- private def showProperties(
- view: View,
- builder: StringBuilder): Unit = {
+ private def showProperties(view: View, builder: StringBuilder): Unit = {
val showProps = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
if (showProps.nonEmpty) {
- val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map {
- case (key, value) =>
- s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
+ val props = conf.redactOptions(showProps).toSeq.sortBy(_._1).map { case (key, value) =>
+ s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
}
builder ++= "TBLPROPERTIES "
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
index 553b808330..ace43eb6c0 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -26,25 +25,22 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog
import org.apache.spark.sql.execution.LeafExecNode
import scala.jdk.CollectionConverters._
-case class ShowV2ViewPropertiesExec(
- output: Seq[Attribute],
- view: View,
- propertyKey: Option[String]) extends V2CommandExec with LeafExecNode {
+case class ShowV2ViewPropertiesExec(output: Seq[Attribute], view: View, propertyKey: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
override protected def run(): Seq[InternalRow] = {
propertyKey match {
case Some(p) =>
- val propValue = properties.getOrElse(p,
- s"View ${view.name()} does not have property: $p")
+ val propValue = properties.getOrElse(p, s"View ${view.name()} does not have property: $p")
Seq(toCatalystRow(p, propValue))
case None =>
- properties.map {
- case (k, v) => toCatalystRow(k, v)
+ properties.map { case (k, v) =>
+ toCatalystRow(k, v)
}.toSeq
}
}
-
private def properties = {
view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
}
diff --git a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
index d82d13a3ed..4e7700b439 100644
--- a/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
+++ b/spark/v4.0/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.sql.catalyst.InternalRow
@@ -28,10 +27,12 @@ import org.apache.spark.sql.internal.SQLConf
import scala.collection.mutable.ArrayBuffer
case class ShowV2ViewsExec(
- output: Seq[Attribute],
- catalog: ViewCatalog,
- namespace: Seq[String],
- pattern: Option[String]) extends V2CommandExec with LeafExecNode {
+ output: Seq[Attribute],
+ catalog: ViewCatalog,
+ namespace: Seq[String],
+ pattern: Option[String])
+ extends V2CommandExec
+ with LeafExecNode {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
@@ -41,7 +42,8 @@ case class ShowV2ViewsExec(
// handle GLOBAL VIEWS
val globalTemp = SQLConf.get.globalTempDatabase
if (namespace.nonEmpty && globalTemp == namespace.head) {
- pattern.map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
+ pattern
+ .map(p => session.sessionState.catalog.globalTempViewManager.listViewNames(p))
.getOrElse(session.sessionState.catalog.globalTempViewManager.listViewNames("*"))
.map(name => rows += toCatalystRow(globalTemp, name, true))
} else {
@@ -54,7 +56,8 @@ case class ShowV2ViewsExec(
}
// include TEMP VIEWS
- pattern.map(p => session.sessionState.catalog.listLocalTempViews(p))
+ pattern
+ .map(p => session.sessionState.catalog.listLocalTempViews(p))
.getOrElse(session.sessionState.catalog.listLocalTempViews("*"))
.map(v => rows += toCatalystRow(v.database.toArray.quoted, v.table, true))
diff --git a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
index 5acaa6800e..5d5990c23b 100644
--- a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
+++ b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.rdd.PartitionCoalescer
@@ -26,10 +25,8 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
// this node doesn't extend RepartitionOperation on purpose to keep this logic isolated
// and ignore it in optimizer rules such as CollapseRepartition
-case class OrderAwareCoalesce(
- numPartitions: Int,
- coalescer: PartitionCoalescer,
- child: LogicalPlan) extends OrderPreservingUnaryNode {
+case class OrderAwareCoalesce(numPartitions: Int, coalescer: PartitionCoalescer, child: LogicalPlan)
+ extends OrderPreservingUnaryNode {
override def output: Seq[Attribute] = child.output
diff --git a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
index 7b599eb3da..3ce3d3e82a 100644
--- a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
+++ b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.iceberg.DistributionMode
@@ -29,16 +28,19 @@ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits
case class SetWriteDistributionAndOrdering(
table: Seq[String],
distributionMode: Option[DistributionMode],
- sortOrder: Seq[(Term, SortDirection, NullOrder)]) extends LeafCommand {
+ sortOrder: Seq[(Term, SortDirection, NullOrder)])
+ extends LeafCommand {
import CatalogV2Implicits._
override lazy val output: Seq[Attribute] = Nil
override def simpleString(maxFields: Int): String = {
- val order = sortOrder.map {
- case (term, direction, nullOrder) => s"$term $direction $nullOrder"
- }.mkString(", ")
+ val order = sortOrder
+ .map { case (term, direction, nullOrder) =>
+ s"$term $direction $nullOrder"
+ }
+ .mkString(", ")
s"SetWriteDistributionAndOrdering ${table.quoted} $distributionMode $order"
}
}
diff --git a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
index bf19ef8a21..71b6107d04 100644
--- a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
+++ b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SortOrderParserUtil.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.iceberg.NullOrder
@@ -27,7 +26,9 @@ import org.apache.iceberg.expressions.Term
class SortOrderParserUtil {
- def collectSortOrder(tableSchema:Schema, sortOrder: Seq[(Term, SortDirection, NullOrder)]): SortOrder = {
+ def collectSortOrder(
+ tableSchema: Schema,
+ sortOrder: Seq[(Term, SortDirection, NullOrder)]): SortOrder = {
val orderBuilder = SortOrder.builderFor(tableSchema)
sortOrder.foreach {
case (term, SortDirection.ASC, nullOrder) =>
diff --git a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
index aa9e9c5533..f46127f818 100644
--- a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
+++ b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.catalyst.utils
import org.apache.iceberg.spark.source.SparkTable
diff --git a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
index 2ef9955052..83dec98c0d 100644
--- a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
+++ b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution
import org.apache.spark.rdd.PartitionCoalescer
@@ -31,7 +30,8 @@ import org.apache.spark.sql.catalyst.plans.physical.UnknownPartitioning
case class OrderAwareCoalesceExec(
numPartitions: Int,
coalescer: PartitionCoalescer,
- child: SparkPlan) extends UnaryExecNode {
+ child: SparkPlan)
+ extends UnaryExecNode {
override def output: Seq[Attribute] = child.output
diff --git a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
index f684ca113c..0be425f7d7 100644
--- a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
+++ b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkExpressionConverter.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.execution.datasources
import org.apache.iceberg.spark.SparkV2Filters
@@ -32,7 +31,8 @@ import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Strategy
object SparkExpressionConverter {
- def convertToIcebergExpression(sparkExpression: Expression): org.apache.iceberg.expressions.Expression = {
+ def convertToIcebergExpression(
+ sparkExpression: Expression): org.apache.iceberg.expressions.Expression = {
// Currently, it is a double conversion as we are converting Spark expression to Spark predicate
// and then converting Spark predicate to Iceberg expression.
// But these two conversions already exist and well tested. So, we are going with this approach.
@@ -40,26 +40,33 @@ object SparkExpressionConverter {
case Some(filter) =>
val converted = SparkV2Filters.convert(filter)
if (converted == null) {
- throw new IllegalArgumentException(s"Cannot convert Spark filter: $filter to Iceberg expression")
+ throw new IllegalArgumentException(
+ s"Cannot convert Spark filter: $filter to Iceberg expression")
}
converted
case _ =>
- throw new IllegalArgumentException(s"Cannot translate Spark expression: $sparkExpression to data source filter")
+ throw new IllegalArgumentException(
+ s"Cannot translate Spark expression: $sparkExpression to data source filter")
}
}
@throws[IcebergAnalysisException]
- def collectResolvedSparkExpression(session: SparkSession, tableName: String, where: String): Expression = {
+ def collectResolvedSparkExpression(
+ session: SparkSession,
+ tableName: String,
+ where: String): Expression = {
val tableAttrs = session.table(tableName).queryExecution.analyzed.output
val unresolvedExpression = session.sessionState.sqlParser.parseExpression(where)
val filter = Filter(unresolvedExpression, DummyRelation(tableAttrs))
val optimizedLogicalPlan = session.sessionState.executePlan(filter).optimizedPlan
- optimizedLogicalPlan.collectFirst {
- case filter: Filter => filter.condition
- case _: DummyRelation => Literal.TrueLiteral
- case _: LocalRelation => Literal.FalseLiteral
- }.getOrElse(throw new IcebergAnalysisException("Failed to find filter expression"))
+ optimizedLogicalPlan
+ .collectFirst {
+ case filter: Filter => filter.condition
+ case _: DummyRelation => Literal.TrueLiteral
+ case _: LocalRelation => Literal.FalseLiteral
+ }
+ .getOrElse(throw new IcebergAnalysisException("Failed to find filter expression"))
}
case class DummyRelation(output: Seq[Attribute]) extends LeafNode
diff --git a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
index b132f7457e..c63f3c9c5c 100644
--- a/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
+++ b/spark/v4.0/spark/src/main/scala/org/apache/spark/sql/stats/ThetaSketchAgg.scala
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.spark.sql.stats
import java.nio.ByteBuffer
@@ -46,12 +45,13 @@ import org.apache.spark.unsafe.types.UTF8String
* which should be deserialized to Compact sketch before using.
*
* See [[https://iceberg.apache.org/puffin-spec/]] for more information.
- *
*/
case class ThetaSketchAgg(
- child: Expression,
- mutableAggBufferOffset: Int = 0,
- inputAggBufferOffset: Int = 0) extends TypedImperativeAggregate[Sketch] with UnaryLike[Expression] {
+ child: Expression,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends TypedImperativeAggregate[Sketch]
+ with UnaryLike[Expression] {
private lazy val icebergType = SparkSchemaUtil.convert(child.dataType)
@@ -102,7 +102,8 @@ case class ThetaSketchAgg(
CompactSketch.wrap(Memory.wrap(storageFormat))
}
- override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate = {
+ override def withNewMutableAggBufferOffset(
+ newMutableAggBufferOffset: Int): ImperativeAggregate = {
copy(mutableAggBufferOffset = newMutableAggBufferOffset)
}