You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2026/08/26 02:25:35 UTC
(hudi) branch master updated: fix(spark): make partition DDL commands honor slash separated date partitioning (#19703)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6cd1196795d8 fix(spark): make partition DDL commands honor slash separated date partitioning (#19703)
6cd1196795d8 is described below
commit 6cd1196795d875adcc935a956e2790a769d39bb0
Author: Sepuri Sai Krishna <sa...@gmail.com>
AuthorDate: Wed Aug 26 07:55:28 2026 +0530
fix(spark): make partition DDL commands honor slash separated date partitioning (#19703)
* fix(spark): make partition DDL commands honor slash separated date partitioning
HoodieSqlCommonUtils#makePartitionPath read hive-style partitioning and URL
encoding from the table config but never getSlashSeparatedDatePartitioning, so
on a slash-partitioned table the DDL commands computed 2026-01-05 while the
writer had laid the partition out as 2026/01/05. ADD PARTITION created a stray
dashed directory (and its existence check never saw the real partition), while
DROP PARTITION and TRUNCATE PARTITION targeted a directory that does not exist
and silently removed nothing.
Apply the same substitution the write path performs, confined to a single
partition field and skipped for hive-style partitioning, with a leading dash
left alone so the partition path never becomes absolute.
Closes #19702
* fix(spark): reuse the write path's path-breaking dash guard in partition DDL
The dash-to-slash substitution these commands perform carried its own guard,
which only refused a leading dash. #19648 widened the write-side rule to reject
any dash-delimited token that is empty, "." or "..", after finding that "..-a"
became "../a" and resolved outside the table base path.
Duplicating the rule is what let the two drift, so ask KeyGenUtils rather than
restate it: the commands have to name the directory the writer created, which
means agreeing on every value, not just on the well formed dates.
Covers the three token classes at the DDL level, where the stray directory is
observable: "..-a" must not create a sibling of the table, "2026-" must not lose
its trailing dash to a normalized-away slash, and "a--b" must not split into
nested directories.
---
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 38 ++++++++-
.../sql/hudi/ddl/TestAlterTableAddPartition.scala | 97 ++++++++++++++++++++++
.../sql/hudi/ddl/TestAlterTableDropPartition.scala | 41 +++++++++
.../spark/sql/hudi/ddl/TestTruncateTable.scala | 35 ++++++++
4 files changed, 208 insertions(+), 3 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
index 1462ceab78dd..9f2027405152 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
@@ -29,6 +29,7 @@ import org.apache.hudi.common.table.timeline.{HoodieInstantTimeGenerator, Hoodie
import org.apache.hudi.common.table.timeline.TimelineUtils.parseDateFromInstantTime
import org.apache.hudi.common.util.PartitionPathEncodeUtils
import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.keygen.KeyGenUtils
import org.apache.hudi.storage.{HoodieStorage, StoragePath, StoragePathInfo}
import org.apache.hudi.util.SparkConfigUtils
@@ -406,24 +407,55 @@ object HoodieSqlCommonUtils extends SparkAdapterSupport {
private def makePartitionPath(partitionFields: Seq[String],
normalizedSpecs: Map[String, String],
enableEncodeUrl: Boolean,
- enableHiveStylePartitioning: Boolean): String = {
+ enableHiveStylePartitioning: Boolean,
+ slashSeparatedDatePartitioning: Boolean): String = {
+ // NOTE: Slash-separated date partitioning only kicks in for a table partitioned by a single
+ // (date) column, mirroring the guard in [[KeyGenUtils#getRecordPartitionPath]] that drives
+ // the write path -- these commands have to name the very same directory the writer created.
+ // Hive-style partitioning is excluded because the config documents the two as mutually
+ // exclusive, and the write paths do not agree on what the combination should produce
+ // (tracked in HUDI issue #19669), so there is no single directory to name here
+ val useSlashSeparatedDates =
+ slashSeparatedDatePartitioning && !enableHiveStylePartitioning && partitionFields.length == 1
partitionFields.map { partitionColumn =>
val encodedPartitionValue = if (enableEncodeUrl) {
PartitionPathEncodeUtils.escapePathName(normalizedSpecs(partitionColumn))
} else {
normalizedSpecs(partitionColumn)
}
- if (enableHiveStylePartitioning) s"$partitionColumn=$encodedPartitionValue" else encodedPartitionValue
+ if (enableHiveStylePartitioning) {
+ s"$partitionColumn=$encodedPartitionValue"
+ } else if (useSlashSeparatedDates) {
+ toSlashSeparatedDate(encodedPartitionValue)
+ } else {
+ encodedPartitionValue
+ }
}.mkString("/")
}
+ /**
+ * Turns a `yyyy-MM-dd` formatted partition value into the `yyyy/MM/dd` directory structure
+ * requested by `hoodie.datasource.write.slash.separated.date.partitioning`, mirroring the
+ * substitution the write path performs in `KeyGenUtils#getRecordPartitionPath`.
+ *
+ * The path-breaking values the writer refuses to slash are refused here for the same reasons, by
+ * asking [[KeyGenUtils#hasPathBreakingDash]] rather than restating the rule: these commands have
+ * to name the directory the writer created, so the two have to agree on every value, not just on
+ * the dates. See that method for what each excluded case does to the path.
+ */
+ private def toSlashSeparatedDate(partitionValue: String): String = {
+ if (KeyGenUtils.hasPathBreakingDash(partitionValue)) partitionValue else partitionValue.replace('-', '/')
+ }
+
def makePartitionPath(hoodieCatalogTable: HoodieCatalogTable,
normalizedSpecs: Map[String, String]): String = {
val tableConfig = hoodieCatalogTable.tableConfig
val enableHiveStylePartitioning = java.lang.Boolean.parseBoolean(tableConfig.getHiveStylePartitioningEnable)
val enableEncodeUrl = java.lang.Boolean.parseBoolean(tableConfig.getUrlEncodePartitioning)
+ val slashSeparatedDatePartitioning = tableConfig.getSlashSeparatedDatePartitioning
- makePartitionPath(hoodieCatalogTable.partitionFields, normalizedSpecs, enableEncodeUrl, enableHiveStylePartitioning)
+ makePartitionPath(hoodieCatalogTable.partitionFields, normalizedSpecs, enableEncodeUrl,
+ enableHiveStylePartitioning, slashSeparatedDatePartitioning)
}
private def validateInstant(queryInstant: String): Unit = {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala
index 3fcb3d4a3e12..0862ab506003 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala
@@ -18,6 +18,9 @@
package org.apache.spark.sql.hudi.ddl
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
+import org.junit.jupiter.api.Assertions.{assertFalse, assertTrue}
+
+import java.io.File
class TestAlterTableAddPartition extends HoodieSparkSqlTestBase {
@@ -227,4 +230,98 @@ class TestAlterTableAddPartition extends HoodieSparkSqlTestBase {
}
}
}
+
+ test("Add partition for a slash separated date partitioned table") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ // create table
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning = 'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+
+ // The writer lays a partition value out as yyyy/MM/dd, so the DDL command has to name that
+ // very directory rather than the dashed value
+ spark.sql(s"""insert into $tableName values (1, "a1", "v1", "2026-01-05")""")
+
+ spark.sql(s"alter table $tableName add partition (dt='2026-02-06')")
+
+ assertTrue(new File(tablePath, "2026/02/06").exists(),
+ "ADD PARTITION should create the slash separated directory")
+ assertFalse(new File(tablePath, "2026-02-06").exists(),
+ "ADD PARTITION should not leave a dashed directory behind")
+
+ // naming the right directory also lets the existence check see the partition the writer created
+ spark.sql(s"alter table $tableName add if not exists partition (dt='2026-01-05')")
+ checkExceptionContain(s"alter table $tableName add partition (dt='2026-01-05')")(
+ "Partition metadata already exists for path")
+ }
+ }
+
+ test("Add partition leaves path breaking values alone under slash separated date partitioning") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning = 'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+
+ // NOTE: The writer refuses to slash these values -- see [[KeyGenUtils#hasPathBreakingDash]]
+ // for what each one does to the path -- so the DDL command has to refuse them too, or
+ // it creates a directory the writer would never have named
+ spark.sql(s"alter table $tableName add partition (dt='..-a')")
+ assertTrue(new File(tablePath, "..-a").exists(),
+ "ADD PARTITION should create the literal directory, not a dot segment")
+ // "../a" would resolve to a sibling of the table directory
+ assertFalse(new File(tmp.getCanonicalPath, "a").exists(),
+ "ADD PARTITION must not create a directory outside the table base path")
+
+ // "2026/" is normalized back to "2026" by StoragePath#normalize
+ spark.sql(s"alter table $tableName add partition (dt='2026-')")
+ assertTrue(new File(tablePath, "2026-").exists(),
+ "ADD PARTITION should keep a trailing dash")
+ assertFalse(new File(tablePath, "2026").exists(),
+ "ADD PARTITION should not drop a trailing dash by way of a trailing slash")
+
+ // "a//b" is collapsed to "a/b" by URI#normalize
+ spark.sql(s"alter table $tableName add partition (dt='a--b')")
+ assertTrue(new File(tablePath, "a--b").exists(),
+ "ADD PARTITION should keep a doubled dash")
+ assertFalse(new File(tablePath, "a").exists(),
+ "ADD PARTITION should not split a doubled dash into nested directories")
+
+ // a single interior dash is still a separator
+ spark.sql(s"alter table $tableName add partition (dt='2026-02-06')")
+ assertTrue(new File(tablePath, "2026/02/06").exists(),
+ "ADD PARTITION should still slash a well formed date")
+ }
+ }
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
index 1e7ac9c4c5ff..8ad58019b0cd 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
@@ -692,4 +692,45 @@ class TestAlterTableDropPartition extends HoodieSparkSqlTestBase {
}
}
}
+
+ test("Drop partition for a slash separated date partitioned table") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ // create table
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning = 'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+ // insert data
+ spark.sql(s"""insert into $tableName values (1, "z3", "v1", "2026-01-05"), (2, "l4", "v1", "2026-01-06")""")
+
+ // the writer laid these out as 2026/01/05 and 2026/01/06, so DROP PARTITION has to name the
+ // same directory -- naming the dashed value drops nothing while still reporting success
+ spark.sql(s"alter table $tableName drop partition (dt='2026-01-05')")
+
+ // trigger clean so that partition deletion kicks in.
+ withSQLConf(HoodieCleanConfig.CLEANER_POLICY.key() -> HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS.name()) {
+ spark.sql(s"call run_clean(table => '$tableName', retain_commits => 1)")
+ .collect()
+ }
+
+ checkAnswer(s"select id, name, ts, dt from $tableName")(
+ Seq(2, "l4", "v1", "2026-01-06")
+ )
+ }
+ }
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala
index ba824c9c68f5..075164a57dff 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala
@@ -148,4 +148,39 @@ class TestTruncateTable extends HoodieSparkSqlTestBase {
}
}
}
+
+ test("Test Truncate partition for a slash separated date partitioned table") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning = 'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+
+ spark.sql(s"""insert into $tableName values (1, "z3", "v1", "2026-01-05"), (2, "l4", "v1", "2026-01-06")""")
+
+ // the writer laid these out as 2026/01/05 and 2026/01/06, so TRUNCATE PARTITION has to name
+ // the same directory -- naming the dashed value truncates nothing
+ spark.sql(s"truncate table $tableName partition (dt='2026-01-05')")
+
+ checkAnswer(s"select id, name, ts, dt from $tableName")(
+ Seq(2, "l4", "v1", "2026-01-06")
+ )
+ }
+ }
}