You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/01/14 19:21:25 UTC
[iceberg] branch master updated (7249d67 -> f1f47bb)

This is an automated email from the ASF dual-hosted git repository.

blue pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git.


    from 7249d67  Spark : Support parallelism in RemoveOrphanFiles (#3872)
     add f1f47bb  Spark 3.2: Implement copy-on-write MERGE (#3804)

No new revisions were added by this update.

Summary of changes:
 .../extensions/IcebergSparkSessionExtensions.scala |   8 +
 .../analysis/AlignRowLevelCommandAssignments.scala |  75 ++++++
 .../analysis/CheckMergeIntoTableConditions.scala   |  74 ++++++
 .../MergeIntoIcebergTableResolutionCheck.scala}    |  21 +-
 .../analysis/ResolveMergeIntoTableReferences.scala | 153 ++++++++++++
 .../catalyst/analysis/RewriteMergeIntoTable.scala  | 274 +++++++++++++++++++++
 .../ExtendedReplaceNullWithFalseInPredicate.scala  |  32 ++-
 .../ExtendedSimplifyConditionalsInPredicate.scala  |  25 +-
 .../IcebergSparkSqlExtensionsParser.scala          |  15 +-
 .../plans/logical/MergeIntoIcebergTable.scala      |  77 ++++++
 .../sql/catalyst/plans/logical/MergeRows.scala}    |  34 ++-
 .../logical/UnresolvedMergeIntoIcebergTable.scala} |  33 +--
 .../v2/ExtendedDataSourceV2Strategy.scala          |   9 +
 .../execution/datasources/v2/MergeRowsExec.scala   | 179 ++++++++++++++
 .../RowLevelCommandDynamicPruning.scala            |   8 +
 .../spark/extensions/TestCopyOnWriteMerge.java     |   2 -
 .../apache/iceberg/spark/extensions/TestMerge.java | 116 +++++----
 .../org/apache/iceberg/spark/SparkWriteConf.java   |  21 +-
 .../iceberg/spark/source/SparkWriteBuilder.java    |   2 +
 19 files changed, 1054 insertions(+), 104 deletions(-)
 create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckMergeIntoTableConditions.scala
 copy spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/{execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala => catalyst/analysis/MergeIntoIcebergTableResolutionCheck.scala} (64%)
 create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoTableReferences.scala
 create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteMergeIntoTable.scala
 create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeIntoIcebergTable.scala
 copy spark/{v3.0/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala => v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala} (59%)
 copy spark/{v3.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala => v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UnresolvedMergeIntoIcebergTable.scala} (55%)
 create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeRowsExec.scala