You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/12/01 07:02:35 UTC
(incubator-paimon) 01/46: [Spark] Truncate Partition (#2371)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.6
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit 4847cc5de8bbd4d4d41c59d62bd57a9beac24e8b
Author: Yann Byron <bi...@gmail.com>
AuthorDate: Thu Nov 23 13:06:56 2023 +0800
[Spark] Truncate Partition (#2371)
---
.../sql/catalyst/analysis/PaimonAnalysis.scala | 16 +++++++++-
.../org/apache/paimon/spark/SparkWriteITCase.java | 34 +++++++++++++++++++---
2 files changed, 45 insertions(+), 5 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/analysis/PaimonAnalysis.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/analysis/PaimonAnalysis.scala
index caca14c73..377539b0a 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/analysis/PaimonAnalysis.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/analysis/PaimonAnalysis.scala
@@ -23,7 +23,7 @@ import org.apache.paimon.table.FileStoreTable
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.analysis.PaimonRelation.isPaimonTable
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, MergeIntoTable, OverwritePartitionsDynamic, PaimonTableValuedFunctions, PaimonTableValueFunction, TruncateTable}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, MergeIntoTable, OverwritePartitionsDynamic, PaimonTableValuedFunctions, PaimonTableValueFunction, TruncatePartition, TruncateTable}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
@@ -50,6 +50,20 @@ case class PaimonPostHocResolutionRules(session: SparkSession) extends Rule[Logi
case t @ TruncateTable(PaimonRelation(table)) if t.resolved =>
PaimonTruncateTableCommand(table, Map.empty)
+ case t @ TruncatePartition(PaimonRelation(table), ResolvedPartitionSpec(names, ident, _))
+ if t.resolved =>
+ assert(names.length == ident.numFields, "Names and values of partition don't match")
+ val resolver = session.sessionState.conf.resolver
+ val schema = table.schema()
+ val partitionSpec = names.zipWithIndex.map {
+ case (name, index) =>
+ val field = schema.find(f => resolver(f.name, name)).getOrElse {
+ throw new RuntimeException(s"$name is not a valid partition column in $schema.")
+ }
+ (name -> ident.get(index, field.dataType).toString)
+ }.toMap
+ PaimonTruncateTableCommand(table, partitionSpec)
+
case _ => plan
}
}
diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java
index 0e930c82f..99c2a97ee 100644
--- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java
+++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java
@@ -129,14 +129,40 @@ public class SparkWriteITCase {
@Test
public void testTruncateTable() {
spark.sql(
- "CREATE TABLE T (a INT, b INT, c STRING) TBLPROPERTIES"
- + " ('primary-key'='a', 'file.format'='avro')");
- spark.sql("INSERT INTO T VALUES (1, 11, '111'), (2, 22, '222')").collectAsList();
- spark.sql("TRUNCATE TABLE T").collectAsList();
+ "CREATE TABLE T (a INT, b INT, c STRING)"
+ + " TBLPROPERTIES ('primary-key'='a', 'file.format'='avro')");
+ spark.sql("INSERT INTO T VALUES (1, 11, '111'), (2, 22, '222')");
+ spark.sql("TRUNCATE TABLE T");
List<Row> rows = spark.sql("SELECT * FROM T").collectAsList();
assertThat(rows.toString()).isEqualTo("[]");
}
+ @Test
+ public void testTruncatePartition1() {
+ spark.sql(
+ "CREATE TABLE T (a INT, b INT, c LONG) PARTITIONED BY (c)"
+ + " TBLPROPERTIES ('primary-key'='a,c')");
+ spark.sql("INSERT INTO T VALUES (1, 11, 111), (2, 22, 222)");
+
+ spark.sql("TRUNCATE TABLE T PARTITION (c = 111)");
+ List<Row> rows = spark.sql("SELECT * FROM T").collectAsList();
+ assertThat(rows.toString()).isEqualTo("[[2,22,222]]");
+ }
+
+ @Test
+ public void testTruncatePartition() {
+ spark.sql(
+ "CREATE TABLE T (a INT, b INT, c LONG, d STRING)"
+ + " PARTITIONED BY (c,d)"
+ + " TBLPROPERTIES ('primary-key'='a,c,d')");
+ spark.sql(
+ "INSERT INTO T VALUES (1, 11, 111, 'a'), (2, 22, 222, 'b'), (3, 33, 333, 'b'), (4, 44, 444, 'a')");
+
+ spark.sql("TRUNCATE TABLE T PARTITION (d = 'a')");
+ List<Row> rows = spark.sql("SELECT * FROM T ORDER BY a").collectAsList();
+ assertThat(rows.toString()).isEqualTo("[[2,22,222,b], [3,33,333,b]]");
+ }
+
@Test
public void testWriteDynamicBucketPartitionedTable() {
spark.sql(