You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ch...@apache.org on 2024/04/19 09:07:24 UTC

(incubator-gluten) branch main updated: [GLUTEN-5454][CH] Support delete/update/optimize/vacuum API for the MergeTree + Delta (#5460)

This is an automated email from the ASF dual-hosted git repository.

changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 24947fa5c [GLUTEN-5454][CH] Support delete/update/optimize/vacuum API for the MergeTree + Delta (#5460)
24947fa5c is described below

commit 24947fa5cb67d8ebc90b603ac03d7ba81e156f8f
Author: Zhichao Zhang <zh...@apache.org>
AuthorDate: Fri Apr 19 17:07:21 2024 +0800

    [GLUTEN-5454][CH] Support delete/update/optimize/vacuum API for the MergeTree + Delta (#5460)
    
    Support delete/update/optimize/vacuum API for the MergeTree + Delta:
    Similar to the DeltaTable API in Delta, add a ClickhouseTable to support delete/update/optimize/vacuum for the MergeTree + Delta by the api way:
    ```
    // create a ClickhouseTable, the dataPath is the delta root path
    val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
    // optimize
    clickhouseTable.optimize().executeCompaction()
    // vacuum
    clickhouseTable.vacuum(0.0)
    // update
    clickhouseTable.updateExpr("l_orderkey = 10086", Map("l_returnflag" -> "'X'"))
    // delete
    clickhouseTable.delete("mod(l_orderkey, 3) = 2")
    ```
    
    Close #5454.
---
 .../delta-20/io/delta/tables/ClickhouseTable.scala |  71 +++++++++++
 ...ommandBase.scala => OptimizeTableCommand.scala} |   7 +-
 .../delta-22/io/delta/tables/ClickhouseTable.scala | 137 +++++++++++++++++++++
 .../sql/delta/commands/OptimizeTableCommand.scala  |   7 +-
 .../spark/sql/delta/commands/CommandUtils.scala    |  43 -------
 .../commands/OptimizeTableCommandOverwrites.scala  |  18 ++-
 .../v2/clickhouse/ClickHouseDataSource.scala       |   3 +-
 .../v2/clickhouse/ClickHouseSparkCatalog.scala     |   2 +-
 .../v2/clickhouse/utils/CHDataSourceUtils.scala    |  78 +++++++++++-
 .../GlutenClickHouseMergeTreeOptimizeSuite.scala   |  34 ++---
 ...tenClickHouseMergeTreePathBasedWriteSuite.scala |  40 +++++-
 ...GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala |   2 +-
 .../GlutenClickHouseMergeTreeWriteOnS3Suite.scala  |   2 +-
 13 files changed, 364 insertions(+), 80 deletions(-)

diff --git a/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala
new file mode 100644
index 000000000..98953fccd
--- /dev/null
+++ b/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.delta.tables
+
+import org.apache.spark.sql.{Dataset, Row, SparkSession}
+import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils}
+import org.apache.spark.sql.delta.catalog.ClickHouseTableV2
+
+import org.apache.hadoop.fs.Path
+
+class ClickhouseTable(
+    @transient private val _df: Dataset[Row],
+    @transient private val table: ClickHouseTableV2)
+  extends DeltaTable(_df, table) {
+
+  override def optimize(): DeltaOptimizeBuilder = {
+    DeltaOptimizeBuilder(
+      sparkSession,
+      table.tableIdentifier.getOrElse(s"clickhouse.`${deltaLog.dataPath.toString}`"))
+  }
+}
+
+object ClickhouseTable {
+
+  /**
+   * Create a DeltaTable for the data at the given `path`.
+   *
+   * Note: This uses the active SparkSession in the current thread to read the table data. Hence,
+   * this throws error if active SparkSession has not been set, that is,
+   * `SparkSession.getActiveSession()` is empty.
+   *
+   * @since 0.3.0
+   */
+  def forPath(path: String): DeltaTable = {
+    val sparkSession = SparkSession.getActiveSession.getOrElse {
+      throw DeltaErrors.activeSparkSessionNotFound()
+    }
+    forPath(sparkSession, path)
+  }
+
+  /**
+   * Create a DeltaTable for the data at the given `path` using the given SparkSession.
+   *
+   * @since 0.3.0
+   */
+  def forPath(sparkSession: SparkSession, path: String): DeltaTable = {
+    val hdpPath = new Path(path)
+    if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath)) {
+      new ClickhouseTable(
+        sparkSession.read.format("clickhouse").load(path),
+        new ClickHouseTableV2(spark = sparkSession, path = hdpPath)
+      )
+    } else {
+      throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path)))
+    }
+  }
+}
diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandBase.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
similarity index 98%
rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandBase.scala
rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
index 2d6ac48f5..571ac63f1 100644
--- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandBase.scala
+++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.delta.skipping.MultiDimClustering
 import org.apache.spark.sql.delta.sources.DeltaSQLConf
 import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand}
 import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts
+import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils
 import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric
 import org.apache.spark.sql.types._
@@ -126,7 +127,7 @@ case class OptimizeTableCommand(
   override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    CommandUtils.ensureClickHouseTableV2(tableId, sparkSession)
+    CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession)
     val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE")
 
     val partitionColumns = deltaLog.snapshot.metadata.partitionColumns
@@ -307,8 +308,8 @@ class OptimizeExecutor(
             // Generally, a bin is a group of existing files, whose total size does not exceed the
             // desired maxFileSize. They will be coalesced into a single output file.
             // However, if isMultiDimClustering = true, all files in a partition will be read by the
-            // same job, the data will be range-partitioned and numFiles = totalFileSize / maxFileSize
-            // will be produced. See below.
+            // same job, the data will be range-partitioned and
+            // numFiles = totalFileSize / maxFileSize will be produced. See below.
             if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) {
               bins += currentBin.toVector
               currentBin.clear()
diff --git a/backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala
new file mode 100644
index 000000000..b5ba0cb44
--- /dev/null
+++ b/backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.delta.tables
+
+import org.apache.spark.sql.{Dataset, Row, SparkSession}
+import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils}
+import org.apache.spark.sql.delta.catalog.ClickHouseTableV2
+
+import org.apache.hadoop.fs.Path
+
+import scala.collection.JavaConverters._
+
+class ClickhouseTable(
+    @transient private val _df: Dataset[Row],
+    @transient private val table: ClickHouseTableV2)
+  extends DeltaTable(_df, table) {
+
+  override def optimize(): DeltaOptimizeBuilder = {
+    DeltaOptimizeBuilder(
+      sparkSession,
+      table.tableIdentifier.getOrElse(s"clickhouse.`${deltaLog.dataPath.toString}`"),
+      table.options)
+  }
+}
+
+object ClickhouseTable {
+
+  /**
+   * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path
+   * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a
+   * `not a Delta table` error.
+   *
+   * Note: This uses the active SparkSession in the current thread to read the table data. Hence,
+   * this throws error if active SparkSession has not been set, that is,
+   * `SparkSession.getActiveSession()` is empty.
+   *
+   * @since 0.3.0
+   */
+  def forPath(path: String): DeltaTable = {
+    val sparkSession = SparkSession.getActiveSession.getOrElse {
+      throw DeltaErrors.activeSparkSessionNotFound()
+    }
+    forPath(sparkSession, path)
+  }
+
+  /**
+   * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path
+   * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a
+   * `not a Delta table` error.
+   *
+   * @since 0.3.0
+   */
+  def forPath(sparkSession: SparkSession, path: String): DeltaTable = {
+    forPath(sparkSession, path, Map.empty[String, String])
+  }
+
+  /**
+   * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path
+   * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a
+   * `not a Delta table` error.
+   *
+   * @param hadoopConf
+   *   Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to
+   *   access the file system when executing queries. Other configurations will not be allowed.
+   *
+   * {{{
+   *   val hadoopConf = Map(
+   *     "fs.s3a.access.key" -> "<access-key>",
+   *     "fs.s3a.secret.key" -> "<secret-key>"
+   *   )
+   *   DeltaTable.forPath(spark, "/path/to/table", hadoopConf)
+   * }}}
+   * @since 2.2.0
+   */
+  def forPath(
+      sparkSession: SparkSession,
+      path: String,
+      hadoopConf: scala.collection.Map[String, String]): DeltaTable = {
+    // We only pass hadoopConf so that we won't pass any unsafe options to Delta.
+    val badOptions = hadoopConf.filterKeys {
+      k => !DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith)
+    }.toMap
+    if (!badOptions.isEmpty) {
+      throw DeltaErrors.unsupportedDeltaTableForPathHadoopConf(badOptions)
+    }
+    val fileSystemOptions: Map[String, String] = hadoopConf.toMap
+    val hdpPath = new Path(path)
+    if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath, fileSystemOptions)) {
+      new ClickhouseTable(
+        sparkSession.read.format("clickhouse").options(fileSystemOptions).load(path),
+        new ClickHouseTableV2(spark = sparkSession, path = hdpPath, options = fileSystemOptions)
+      )
+    } else {
+      throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path)))
+    }
+  }
+
+  /**
+   * Java friendly API to instantiate a [[DeltaTable]] object representing the data at the given
+   * path, If the given path is invalid (i.e. either no table exists or an existing table is not a
+   * Delta table), it throws a `not a Delta table` error.
+   *
+   * @param hadoopConf
+   *   Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to
+   *   access the file system when executing queries. Other configurations will be ignored.
+   *
+   * {{{
+   *   val hadoopConf = Map(
+   *     "fs.s3a.access.key" -> "<access-key>",
+   *     "fs.s3a.secret.key", "<secret-key>"
+   *   )
+   *   DeltaTable.forPath(spark, "/path/to/table", hadoopConf)
+   * }}}
+   * @since 2.2.0
+   */
+  def forPath(
+      sparkSession: SparkSession,
+      path: String,
+      hadoopConf: java.util.Map[String, String]): DeltaTable = {
+    val fsOptions = hadoopConf.asScala.toMap
+    forPath(sparkSession, path, fsOptions)
+  }
+}
diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
index 346943671..e69b3aa65 100644
--- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
+++ b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.delta.skipping.MultiDimClustering
 import org.apache.spark.sql.delta.sources.DeltaSQLConf
 import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand}
 import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts
+import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils
 import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric
 import org.apache.spark.sql.types._
@@ -126,7 +127,7 @@ case class OptimizeTableCommand(
   override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    CommandUtils.ensureClickHouseTableV2(tableId, sparkSession)
+    CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession)
 
     val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE", options)
 
@@ -295,8 +296,8 @@ class OptimizeExecutor(
             // Generally, a bin is a group of existing files, whose total size does not exceed the
             // desired maxFileSize. They will be coalesced into a single output file.
             // However, if isMultiDimClustering = true, all files in a partition will be read by the
-            // same job, the data will be range-partitioned and numFiles = totalFileSize / maxFileSize
-            // will be produced. See below.
+            // same job, the data will be range-partitioned and
+            // umFiles = totalFileSize / maxFileSize will be produced. See below.
             if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) {
               bins += currentBin.toVector
               currentBin.clear()
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/CommandUtils.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/CommandUtils.scala
deleted file mode 100644
index 262c37eff..000000000
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/CommandUtils.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.delta.commands
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.connector.catalog.Identifier
-
-object CommandUtils {
-  // Ensure ClickHouseTableV2 table exists
-  def ensureClickHouseTableV2(
-      tableId: Option[TableIdentifier],
-      sparkSession: SparkSession): Unit = {
-    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
-    if (tableId.isEmpty) {
-      throw new UnsupportedOperationException("Current command requires table identifier.")
-    }
-    // If user comes into this function without previously triggering loadTable
-    // (which creates ClickhouseTableV2), we have to load the table manually
-    // Notice: Multi-catalog case is not well considered!
-    sparkSession.sessionState.catalogManager.currentCatalog.asTableCatalog.loadTable(
-      Identifier.of(
-        Array(
-          tableId.get.database.getOrElse(
-            sparkSession.sessionState.catalogManager.currentNamespace.head)),
-        tableId.get.table)
-    )
-  }
-}
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
index 36ce1637b..7af1abe43 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
@@ -25,7 +25,7 @@ import org.apache.spark.internal.io.SparkHadoopWriterUtils
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
-import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, DeltaTableIdentifier, OptimisticTransaction}
+import org.apache.spark.sql.delta._
 import org.apache.spark.sql.delta.actions.{AddFile, FileAction}
 import org.apache.spark.sql.delta.catalog.ClickHouseTableV2
 import org.apache.spark.sql.errors.QueryExecutionErrors
@@ -33,6 +33,7 @@ import org.apache.spark.sql.execution.datasources.CHDatasourceJniWrapper
 import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects
 import org.apache.spark.sql.execution.datasources.v1.clickhouse._
 import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts}
+import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils}
@@ -264,12 +265,21 @@ object OptimizeTableCommandOverwrites extends Logging {
       operationName: String,
       hadoopConf: Map[String, String] = Map.empty): DeltaLog = {
     val tablePath =
-      if (tableIdentifier.nonEmpty && isDeltaTable(spark, tableIdentifier.get)) {
+      if (path.nonEmpty) {
+        new Path(path.get)
+      } else if (tableIdentifier.nonEmpty) {
         val sessionCatalog = spark.sessionState.catalog
         lazy val metadata = sessionCatalog.getTableMetadata(tableIdentifier.get)
-        new Path(metadata.location)
+
+        if (CHDataSourceUtils.isClickhousePath(spark, tableIdentifier.get)) {
+          new Path(tableIdentifier.get.table)
+        } else if (CHDataSourceUtils.isClickHouseTable(spark, tableIdentifier.get)) {
+          new Path(metadata.location)
+        } else {
+          throw DeltaErrors.notADeltaTableException(operationName)
+        }
       } else {
-        throw new UnsupportedOperationException("OPTIMIZE is ony supported for clickhouse tables")
+        throw DeltaErrors.missingTableIdentifierException(operationName)
       }
 
     val startTime = Some(System.currentTimeMillis)
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala
index 4f96ef812..8c1062f4c 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala
@@ -72,7 +72,8 @@ class ClickHouseDataSource extends DeltaDataSource {
       .getOrElse(Nil)
 
     val deltaLog = DeltaLog.forTable(sqlContext.sparkSession, path, parameters)
-    val configs = if (DeltaLogAdapter.snapshot(deltaLog).version < 0) {
+    // need to use the latest snapshot
+    val configs = if (deltaLog.update().version < 0) {
       // when creating table, save the clickhouse config to the delta metadata
       val clickHouseTableV2 = ClickHouseTableV2.getTable(deltaLog)
       clickHouseTableV2.properties().asScala.toMap ++ DeltaConfigs
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala
index daaec00a7..c2d208df5 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala
@@ -227,7 +227,7 @@ class ClickHouseSparkCatalog
   override def loadTable(ident: Identifier): Table = {
     try {
       super.loadTable(ident) match {
-        case v1: V1Table if CHDataSourceUtils.isDeltaTable(v1.catalogTable) =>
+        case v1: V1Table if CHDataSourceUtils.isClickHouseTable(v1.catalogTable) =>
           new ClickHouseTableV2(
             spark,
             new Path(v1.catalogTable.location),
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala
index 190315713..e2e7ac491 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala
@@ -16,8 +16,12 @@
  */
 package org.apache.spark.sql.execution.datasources.v2.clickhouse.utils
 
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{AnalysisException, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError
 import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig
 
 import org.apache.hadoop.fs.Path
@@ -26,10 +30,11 @@ import java.util.Locale
 
 import scala.util.Try
 
-object CHDataSourceUtils {
+object CHDataSourceUtils extends Logging {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
   /** Check whether this table is a Delta table based on information from the Catalog. */
-  def isDeltaTable(table: CatalogTable): Boolean =
+  def isClickHouseTable(table: CatalogTable): Boolean =
     CHDataSourceUtils.isClickHouseTable(table.provider)
 
   /** Check whether this table is a ClickHouse table based on information from the Catalog. */
@@ -42,6 +47,48 @@ object CHDataSourceUtils {
     name.toLowerCase(Locale.ROOT) == ClickHouseConfig.ALT_NAME
   }
 
+  /**
+   * Check whether the provided table name is a Clickhouse table based on information from the
+   * Catalog.
+   */
+  def isClickHouseTable(spark: SparkSession, tableName: TableIdentifier): Boolean = {
+    val catalog = spark.sessionState.catalog
+    val tableIsNotTemporaryTable = !catalog.isTempView(tableName)
+    val tableExists = {
+      (tableName.database.isEmpty || catalog.databaseExists(tableName.database.get)) &&
+      catalog.tableExists(tableName)
+    }
+    tableIsNotTemporaryTable &&
+    tableExists &&
+    isClickHouseTable(catalog.getTableMetadata(tableName))
+  }
+
+  /** Check the specified table identifier represents a Clickhouse path. */
+  def isClickhousePath(spark: SparkSession, identifier: TableIdentifier): Boolean = {
+    val catalog = spark.sessionState.catalog
+
+    def tableIsTemporaryTable = catalog.isTempView(identifier)
+
+    def tableExists: Boolean = {
+      try {
+        catalog.databaseExists(identifier.database.get) && catalog.tableExists(identifier)
+      } catch {
+        case e: AnalysisException if gluePermissionError(e) =>
+          logWarning(
+            "Received an access denied error from Glue. Will check to see if this " +
+              s"identifier ($identifier) is path based.",
+            e)
+          false
+      }
+    }
+
+    spark.sessionState.conf.runSQLonFile &&
+    isClickHouseTable(identifier.database) &&
+    !tableIsTemporaryTable &&
+    !tableExists &&
+    new Path(identifier.table).isAbsolute
+  }
+
   /** Find the root of a Delta table from the provided path. */
   def findClickHouseTableRoot(
       spark: SparkSession,
@@ -49,7 +96,10 @@ object CHDataSourceUtils {
       options: Map[String, String] = Map.empty): Option[Path] = {
     val fs = path.getFileSystem(spark.sessionState.newHadoopConfWithOptions(options))
     var currentPath = path
-    while (currentPath != null && currentPath.getName != ClickHouseConfig.METADATA_DIR) {
+    while (
+      currentPath != null && currentPath.getName != ClickHouseConfig.METADATA_DIR &&
+      currentPath.getName != "_samples"
+    ) {
       val deltaLogPath = new Path(currentPath, ClickHouseConfig.METADATA_DIR)
       if (Try(fs.exists(deltaLogPath)).getOrElse(false)) {
         return Option(currentPath)
@@ -58,4 +108,24 @@ object CHDataSourceUtils {
     }
     None
   }
+
+  // Ensure ClickHouseTableV2 table exists
+  def ensureClickHouseTableV2(
+      tableId: Option[TableIdentifier],
+      sparkSession: SparkSession): Unit = {
+    if (tableId.isEmpty) {
+      throw new UnsupportedOperationException("Current command requires table identifier.")
+    }
+    // If user comes into this function without previously triggering loadTable
+    // (which creates ClickhouseTableV2), we have to load the table manually
+    // Notice: Multi-catalog case is not well considered!
+    val table = sparkSession.sessionState.catalogManager.currentCatalog.asTableCatalog.loadTable(
+      Identifier.of(
+        Array(
+          tableId.get.database.getOrElse(
+            sparkSession.sessionState.catalogManager.currentNamespace.head)),
+        tableId.get.table)
+    )
+    table.name()
+  }
 }
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
index c96058f55..9635b9958 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
@@ -20,6 +20,8 @@ import org.apache.spark.SparkConf
 import org.apache.spark.sql.SaveMode
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 
+import io.delta.tables.ClickhouseTable
+
 import java.io.File
 
 // Some sqls' line length exceeds 100
@@ -361,9 +363,8 @@ class GlutenClickHouseMergeTreeOptimizeSuite
     }
   }
 
-  // TODO: support later
-  ignore("test mergetree optimize with the path based table") {
-    val dataPath = s"$basePath/lineitem_mergetree_optimize_p1"
+  test("test mergetree optimize with the path based table") {
+    val dataPath = s"$basePath/lineitem_mergetree_optimize_path_based"
     clearDataPath(dataPath)
     withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "838000") {
       // 3 from 37 parts are larger than this, so after optimize there should be 4 parts:
@@ -378,15 +379,16 @@ class GlutenClickHouseMergeTreeOptimizeSuite
         .mode(SaveMode.Append)
         .save(dataPath)
 
-      spark.sql(s"optimize clickhouse.`$dataPath`")
+      val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+      clickhouseTable.optimize().executeCompaction()
 
       spark.sql("set spark.gluten.enabled=false")
-      spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
-      spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
+      clickhouseTable.vacuum(0.0)
+      clickhouseTable.vacuum(0.0)
       spark.sql("set spark.gluten.enabled=true")
       assert(countFiles(new File(dataPath)) == 99)
 
-      val ret = spark.sql("select count(*) from clickhouse.`${dataPath}`").collect()
+      val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect()
       assert(ret.apply(0).get(0) == 600572)
     }
 
@@ -398,28 +400,30 @@ class GlutenClickHouseMergeTreeOptimizeSuite
       // 1 merged part from 2 original parts, 1 merged part from 34 original parts
       // and 1 original part (size 838255)
 
-      spark.sql("optimize clickhouse.`${dataPath}`")
+      val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+      clickhouseTable.optimize().executeCompaction()
 
       spark.sql("set spark.gluten.enabled=false")
-      spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
-      spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
+      clickhouseTable.vacuum(0.0)
+      clickhouseTable.vacuum(0.0)
       spark.sql("set spark.gluten.enabled=true")
       assert(countFiles(new File(dataPath)) == 93)
 
-      val ret = spark.sql("select count(*) from clickhouse.`${dataPath}`").collect()
+      val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect()
       assert(ret.apply(0).get(0) == 600572)
     }
 
     // now merge all parts (testing merging from merged parts)
-    spark.sql("optimize clickhouse.`${dataPath}`")
+    val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+    clickhouseTable.optimize().executeCompaction()
 
     spark.sql("set spark.gluten.enabled=false")
-    spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
-    spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
+    clickhouseTable.vacuum(0.0)
+    clickhouseTable.vacuum(0.0)
     spark.sql("set spark.gluten.enabled=true")
     assert(countFiles(new File(dataPath)) == 77)
 
-    val ret = spark.sql("select count(*) from clickhouse.`${dataPath}`").collect()
+    val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect()
     assert(ret.apply(0).get(0) == 600572)
   }
 }
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala
index dcaf69ee4..d85217536 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts
 import org.apache.spark.sql.functions._
 
+import io.delta.tables.ClickhouseTable
 import org.apache.commons.io.filefilter.WildcardFileFilter
 
 import java.io.File
@@ -405,6 +406,37 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite
       assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted))
     }
 
+    val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+    clickhouseTable.updateExpr("l_orderkey = 10086", Map("l_returnflag" -> "'X'"))
+
+    {
+      val df = spark.read
+        .format("clickhouse")
+        .load(dataPath)
+        .where("l_returnflag = 'X'")
+      assert(df.count() == 1)
+      val scanExec = collect(df.queryExecution.executedPlan) {
+        case f: FileSourceScanExecTransformer => f
+      }
+      assert(scanExec.size == 1)
+
+      val mergetreeScan = scanExec.head
+      assert(mergetreeScan.nodeName.startsWith("Scan mergetree"))
+
+      val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex]
+      val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts])
+      assert(
+        addFiles.map(_.rows).sum
+          == 600572)
+
+      // 4 parts belong to the first batch
+      // 2 parts belong to the second batch (1 actual updated part, 1 passively updated).
+      assert(addFiles.size == 6)
+      val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_")))
+      assert(filePaths.size == 2)
+      assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted))
+    }
+
     val df = spark.read
       .format("clickhouse")
       .load(dataPath)
@@ -444,9 +476,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite
     assert(filePaths.size == 2)
     assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted))
 
-    spark.sql(s"""
-                 | delete from clickhouse.`$dataPath` where mod(l_orderkey, 3) = 2
-                 |""".stripMargin)
+    val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+    clickhouseTable.delete("mod(l_orderkey, 3) = 2")
     val df1 = spark.read
       .format("clickhouse")
       .load(dataPath)
@@ -1272,7 +1303,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite
 
     checkQueryResult(dataPath)
 
-    val dataPath1 = s"$basePath/lineitem_mergetree_ctas_5219_1"
+    val dataPath1 = s"$basePath/lineitem_mergetree_ctas_5219"
+    clearDataPath(dataPath1)
     sourceDF.write
       .format("clickhouse")
       .mode(SaveMode.Append)
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala
index 7f507af23..49011e031 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala
@@ -535,7 +535,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite
   }
 
   ignore("test mergetree write with the path based") {
-    val dataPath = s"$HDFS_URL/test/lineitem_mergetree_bucket_path_based_hdfs"
+    val dataPath = s"$HDFS_URL/test/lineitem_mergetree_bucket_hdfs"
 
     val sourceDF = spark.sql(s"""
                                 |select * from lineitem
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala
index 95df0a086..2b6414388 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala
@@ -552,7 +552,7 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite
   }
 
   test("test mergetree write with the path based") {
-    val dataPath = s"s3a://$BUCKET_NAME/lineitem_mergetree_bucket_path_based_s3"
+    val dataPath = s"s3a://$BUCKET_NAME/lineitem_mergetree_bucket_s3"
 
     val sourceDF = spark.sql(s"""
                                 |select * from lineitem


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org