You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/12/17 00:26:03 UTC
[iceberg] branch master updated (339866d -> a2260fa)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git.
from 339866d Hive: ORC vectorization fails when split offsets are considered during split generation (#3748)
add a2260fa Spark: Implement copy-on-write DELETE (#3661)
No new revisions were added by this update.
Summary of changes:
.baseline/checkstyle/checkstyle.xml | 1 +
.../java/org/apache/iceberg/IsolationLevel.java | 10 +-
.../org/apache/iceberg/RowLevelOperationMode.java | 53 ++++
.../extensions/IcebergSparkSessionExtensions.scala | 18 ++
.../catalyst/analysis/RewriteDeleteFromTable.scala | 91 +++++++
.../catalyst/analysis/RewriteRowLevelCommand.scala | 65 +++++
.../ExtendedReplaceNullWithFalseInPredicate.scala | 102 +++++++
.../ExtendedSimplifyConditionalsInPredicate.scala | 72 +++++
.../IcebergSparkSqlExtensionsParser.scala | 57 +++-
.../planning/RewrittenRowLevelCommand.scala | 87 ++++++
.../plans/logical/DeleteFromIcebergTable.scala | 49 ++++
.../sql/catalyst/plans/logical/ReplaceData.scala | 69 +++++
.../catalyst/plans/logical/RowLevelCommand.scala} | 23 +-
.../plans/logical/V2WriteCommandLike.scala | 42 +++
.../write/ExtendedLogicalWriteInfoImpl.scala} | 28 +-
.../write/RowLevelOperationInfoImpl.scala} | 25 +-
.../connector/write/RowLevelOperationTable.scala | 54 ++++
.../v2/ExtendedDataSourceV2Implicits.scala} | 33 +--
.../v2/ExtendedDataSourceV2Strategy.scala | 32 ++-
.../datasources/v2/ExtendedV2Writes.scala | 33 ++-
.../v2/OptimizeMetadataOnlyDeleteFromTable.scala | 69 +++++
.../datasources/v2/ReplaceDataExec.scala} | 29 +-
.../v2/ReplaceRewrittenRowLevelCommand.scala} | 27 +-
.../v2/RowLevelCommandScanRelationPushDown.scala | 81 ++++++
.../RowLevelCommandDynamicPruning.scala | 134 ++++++++++
.../spark/extensions/SparkExtensionsTestBase.java | 5 +
.../spark/extensions/TestCopyOnWriteDelete.java | 2 -
.../iceberg/spark/extensions/TestDelete.java | 80 ++++--
.../spark/SparkDistributionAndOrderingUtil.java | 34 +++
.../org/apache/iceberg/spark/SparkWriteConf.java | 21 +-
.../iceberg/spark/source/SparkBatchScan.java | 9 +-
.../spark/source/SparkCopyOnWriteOperation.java | 102 +++++++
...arkMergeScan.java => SparkCopyOnWriteScan.java} | 113 ++++----
.../source/SparkRowLevelOperationBuilder.java | 110 ++++++++
.../iceberg/spark/source/SparkScanBuilder.java | 53 +++-
.../apache/iceberg/spark/source/SparkTable.java | 10 +-
.../apache/iceberg/spark/source/SparkWrite.java | 10 +-
.../iceberg/spark/source/SparkWriteBuilder.java | 31 ++-
.../catalog/SupportsRowLevelOperations.java | 39 +++
.../TestSparkDistributionAndOrderingUtil.java | 293 +++++++++++++++++++++
40 files changed, 1979 insertions(+), 217 deletions(-)
create mode 100644 core/src/main/java/org/apache/iceberg/RowLevelOperationMode.java
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDeleteFromTable.scala
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteRowLevelCommand.scala
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedReplaceNullWithFalseInPredicate.scala
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ExtendedSimplifyConditionalsInPredicate.scala
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/planning/RewrittenRowLevelCommand.scala
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromIcebergTable.scala
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala
copy spark/v3.2/spark-extensions/src/{test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java => main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala} (53%)
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/V2WriteCommandLike.scala
copy spark/v3.2/spark-extensions/src/{test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java => main/scala/org/apache/spark/sql/connector/write/ExtendedLogicalWriteInfoImpl.scala} (53%)
copy spark/v3.2/spark-extensions/src/{test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java => main/scala/org/apache/spark/sql/connector/write/RowLevelOperationInfoImpl.scala} (53%)
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationTable.scala
copy spark/v3.2/spark-extensions/src/{test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java => main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala} (53%)
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/OptimizeMetadataOnlyDeleteFromTable.scala
copy spark/v3.2/spark-extensions/src/{test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java => main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala} (53%)
copy spark/v3.2/spark-extensions/src/{test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java => main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceRewrittenRowLevelCommand.scala} (53%)
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RowLevelCommandScanRelationPushDown.scala
create mode 100644 spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/RowLevelCommandDynamicPruning.scala
create mode 100644 spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkCopyOnWriteOperation.java
rename spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/{SparkMergeScan.java => SparkCopyOnWriteScan.java} (55%)
create mode 100644 spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkRowLevelOperationBuilder.java
create mode 100644 spark/v3.2/spark/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsRowLevelOperations.java