You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ch...@apache.org on 2024/04/19 09:07:24 UTC
(incubator-gluten) branch main updated: [GLUTEN-5454][CH] Support delete/update/optimize/vacuum API for the MergeTree + Delta (#5460)
This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 24947fa5c [GLUTEN-5454][CH] Support delete/update/optimize/vacuum API for the MergeTree + Delta (#5460)
24947fa5c is described below
commit 24947fa5cb67d8ebc90b603ac03d7ba81e156f8f
Author: Zhichao Zhang <zh...@apache.org>
AuthorDate: Fri Apr 19 17:07:21 2024 +0800
[GLUTEN-5454][CH] Support delete/update/optimize/vacuum API for the MergeTree + Delta (#5460)
Support delete/update/optimize/vacuum API for the MergeTree + Delta:
Similar to the DeltaTable API in Delta, add a ClickhouseTable to support delete/update/optimize/vacuum for the MergeTree + Delta by the api way:
```
// create a ClickhouseTable, the dataPath is the delta root path
val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
// optimize
clickhouseTable.optimize().executeCompaction()
// vacuum
clickhouseTable.vacuum(0.0)
// update
clickhouseTable.updateExpr("l_orderkey = 10086", Map("l_returnflag" -> "'X'"))
// delete
clickhouseTable.delete("mod(l_orderkey, 3) = 2")
```
Close #5454.
---
.../delta-20/io/delta/tables/ClickhouseTable.scala | 71 +++++++++++
...ommandBase.scala => OptimizeTableCommand.scala} | 7 +-
.../delta-22/io/delta/tables/ClickhouseTable.scala | 137 +++++++++++++++++++++
.../sql/delta/commands/OptimizeTableCommand.scala | 7 +-
.../spark/sql/delta/commands/CommandUtils.scala | 43 -------
.../commands/OptimizeTableCommandOverwrites.scala | 18 ++-
.../v2/clickhouse/ClickHouseDataSource.scala | 3 +-
.../v2/clickhouse/ClickHouseSparkCatalog.scala | 2 +-
.../v2/clickhouse/utils/CHDataSourceUtils.scala | 78 +++++++++++-
.../GlutenClickHouseMergeTreeOptimizeSuite.scala | 34 ++---
...tenClickHouseMergeTreePathBasedWriteSuite.scala | 40 +++++-
...GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala | 2 +-
.../GlutenClickHouseMergeTreeWriteOnS3Suite.scala | 2 +-
13 files changed, 364 insertions(+), 80 deletions(-)
diff --git a/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala
new file mode 100644
index 000000000..98953fccd
--- /dev/null
+++ b/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.delta.tables
+
+import org.apache.spark.sql.{Dataset, Row, SparkSession}
+import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils}
+import org.apache.spark.sql.delta.catalog.ClickHouseTableV2
+
+import org.apache.hadoop.fs.Path
+
+class ClickhouseTable(
+ @transient private val _df: Dataset[Row],
+ @transient private val table: ClickHouseTableV2)
+ extends DeltaTable(_df, table) {
+
+ override def optimize(): DeltaOptimizeBuilder = {
+ DeltaOptimizeBuilder(
+ sparkSession,
+ table.tableIdentifier.getOrElse(s"clickhouse.`${deltaLog.dataPath.toString}`"))
+ }
+}
+
+object ClickhouseTable {
+
+ /**
+ * Create a DeltaTable for the data at the given `path`.
+ *
+ * Note: This uses the active SparkSession in the current thread to read the table data. Hence,
+ * this throws error if active SparkSession has not been set, that is,
+ * `SparkSession.getActiveSession()` is empty.
+ *
+ * @since 0.3.0
+ */
+ def forPath(path: String): DeltaTable = {
+ val sparkSession = SparkSession.getActiveSession.getOrElse {
+ throw DeltaErrors.activeSparkSessionNotFound()
+ }
+ forPath(sparkSession, path)
+ }
+
+ /**
+ * Create a DeltaTable for the data at the given `path` using the given SparkSession.
+ *
+ * @since 0.3.0
+ */
+ def forPath(sparkSession: SparkSession, path: String): DeltaTable = {
+ val hdpPath = new Path(path)
+ if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath)) {
+ new ClickhouseTable(
+ sparkSession.read.format("clickhouse").load(path),
+ new ClickHouseTableV2(spark = sparkSession, path = hdpPath)
+ )
+ } else {
+ throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path)))
+ }
+ }
+}
diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandBase.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
similarity index 98%
rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandBase.scala
rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
index 2d6ac48f5..571ac63f1 100644
--- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandBase.scala
+++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.delta.skipping.MultiDimClustering
import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand}
import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts
+import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric
import org.apache.spark.sql.types._
@@ -126,7 +127,7 @@ case class OptimizeTableCommand(
override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil
override def run(sparkSession: SparkSession): Seq[Row] = {
- CommandUtils.ensureClickHouseTableV2(tableId, sparkSession)
+ CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession)
val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE")
val partitionColumns = deltaLog.snapshot.metadata.partitionColumns
@@ -307,8 +308,8 @@ class OptimizeExecutor(
// Generally, a bin is a group of existing files, whose total size does not exceed the
// desired maxFileSize. They will be coalesced into a single output file.
// However, if isMultiDimClustering = true, all files in a partition will be read by the
- // same job, the data will be range-partitioned and numFiles = totalFileSize / maxFileSize
- // will be produced. See below.
+ // same job, the data will be range-partitioned and
+ // numFiles = totalFileSize / maxFileSize will be produced. See below.
if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) {
bins += currentBin.toVector
currentBin.clear()
diff --git a/backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala
new file mode 100644
index 000000000..b5ba0cb44
--- /dev/null
+++ b/backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.delta.tables
+
+import org.apache.spark.sql.{Dataset, Row, SparkSession}
+import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils}
+import org.apache.spark.sql.delta.catalog.ClickHouseTableV2
+
+import org.apache.hadoop.fs.Path
+
+import scala.collection.JavaConverters._
+
+class ClickhouseTable(
+ @transient private val _df: Dataset[Row],
+ @transient private val table: ClickHouseTableV2)
+ extends DeltaTable(_df, table) {
+
+ override def optimize(): DeltaOptimizeBuilder = {
+ DeltaOptimizeBuilder(
+ sparkSession,
+ table.tableIdentifier.getOrElse(s"clickhouse.`${deltaLog.dataPath.toString}`"),
+ table.options)
+ }
+}
+
+object ClickhouseTable {
+
+ /**
+ * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path
+ * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a
+ * `not a Delta table` error.
+ *
+ * Note: This uses the active SparkSession in the current thread to read the table data. Hence,
+ * this throws error if active SparkSession has not been set, that is,
+ * `SparkSession.getActiveSession()` is empty.
+ *
+ * @since 0.3.0
+ */
+ def forPath(path: String): DeltaTable = {
+ val sparkSession = SparkSession.getActiveSession.getOrElse {
+ throw DeltaErrors.activeSparkSessionNotFound()
+ }
+ forPath(sparkSession, path)
+ }
+
+ /**
+ * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path
+ * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a
+ * `not a Delta table` error.
+ *
+ * @since 0.3.0
+ */
+ def forPath(sparkSession: SparkSession, path: String): DeltaTable = {
+ forPath(sparkSession, path, Map.empty[String, String])
+ }
+
+ /**
+ * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path
+ * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a
+ * `not a Delta table` error.
+ *
+ * @param hadoopConf
+ * Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to
+ * access the file system when executing queries. Other configurations will not be allowed.
+ *
+ * {{{
+ * val hadoopConf = Map(
+ * "fs.s3a.access.key" -> "<access-key>",
+ * "fs.s3a.secret.key" -> "<secret-key>"
+ * )
+ * DeltaTable.forPath(spark, "/path/to/table", hadoopConf)
+ * }}}
+ * @since 2.2.0
+ */
+ def forPath(
+ sparkSession: SparkSession,
+ path: String,
+ hadoopConf: scala.collection.Map[String, String]): DeltaTable = {
+ // We only pass hadoopConf so that we won't pass any unsafe options to Delta.
+ val badOptions = hadoopConf.filterKeys {
+ k => !DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith)
+ }.toMap
+ if (!badOptions.isEmpty) {
+ throw DeltaErrors.unsupportedDeltaTableForPathHadoopConf(badOptions)
+ }
+ val fileSystemOptions: Map[String, String] = hadoopConf.toMap
+ val hdpPath = new Path(path)
+ if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath, fileSystemOptions)) {
+ new ClickhouseTable(
+ sparkSession.read.format("clickhouse").options(fileSystemOptions).load(path),
+ new ClickHouseTableV2(spark = sparkSession, path = hdpPath, options = fileSystemOptions)
+ )
+ } else {
+ throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path)))
+ }
+ }
+
+ /**
+ * Java friendly API to instantiate a [[DeltaTable]] object representing the data at the given
+ * path, If the given path is invalid (i.e. either no table exists or an existing table is not a
+ * Delta table), it throws a `not a Delta table` error.
+ *
+ * @param hadoopConf
+ * Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to
+ * access the file system when executing queries. Other configurations will be ignored.
+ *
+ * {{{
+ * val hadoopConf = Map(
+ * "fs.s3a.access.key" -> "<access-key>",
+ * "fs.s3a.secret.key", "<secret-key>"
+ * )
+ * DeltaTable.forPath(spark, "/path/to/table", hadoopConf)
+ * }}}
+ * @since 2.2.0
+ */
+ def forPath(
+ sparkSession: SparkSession,
+ path: String,
+ hadoopConf: java.util.Map[String, String]): DeltaTable = {
+ val fsOptions = hadoopConf.asScala.toMap
+ forPath(sparkSession, path, fsOptions)
+ }
+}
diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
index 346943671..e69b3aa65 100644
--- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
+++ b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.delta.skipping.MultiDimClustering
import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand}
import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts
+import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric
import org.apache.spark.sql.types._
@@ -126,7 +127,7 @@ case class OptimizeTableCommand(
override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil
override def run(sparkSession: SparkSession): Seq[Row] = {
- CommandUtils.ensureClickHouseTableV2(tableId, sparkSession)
+ CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession)
val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE", options)
@@ -295,8 +296,8 @@ class OptimizeExecutor(
// Generally, a bin is a group of existing files, whose total size does not exceed the
// desired maxFileSize. They will be coalesced into a single output file.
// However, if isMultiDimClustering = true, all files in a partition will be read by the
- // same job, the data will be range-partitioned and numFiles = totalFileSize / maxFileSize
- // will be produced. See below.
+ // same job, the data will be range-partitioned and
+ // umFiles = totalFileSize / maxFileSize will be produced. See below.
if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) {
bins += currentBin.toVector
currentBin.clear()
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/CommandUtils.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/CommandUtils.scala
deleted file mode 100644
index 262c37eff..000000000
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/CommandUtils.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.delta.commands
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.connector.catalog.Identifier
-
-object CommandUtils {
- // Ensure ClickHouseTableV2 table exists
- def ensureClickHouseTableV2(
- tableId: Option[TableIdentifier],
- sparkSession: SparkSession): Unit = {
- import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
- if (tableId.isEmpty) {
- throw new UnsupportedOperationException("Current command requires table identifier.")
- }
- // If user comes into this function without previously triggering loadTable
- // (which creates ClickhouseTableV2), we have to load the table manually
- // Notice: Multi-catalog case is not well considered!
- sparkSession.sessionState.catalogManager.currentCatalog.asTableCatalog.loadTable(
- Identifier.of(
- Array(
- tableId.get.database.getOrElse(
- sparkSession.sessionState.catalogManager.currentNamespace.head)),
- tableId.get.table)
- )
- }
-}
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
index 36ce1637b..7af1abe43 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala
@@ -25,7 +25,7 @@ import org.apache.spark.internal.io.SparkHadoopWriterUtils
import org.apache.spark.shuffle.FetchFailedException
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
-import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, DeltaTableIdentifier, OptimisticTransaction}
+import org.apache.spark.sql.delta._
import org.apache.spark.sql.delta.actions.{AddFile, FileAction}
import org.apache.spark.sql.delta.catalog.ClickHouseTableV2
import org.apache.spark.sql.errors.QueryExecutionErrors
@@ -33,6 +33,7 @@ import org.apache.spark.sql.execution.datasources.CHDatasourceJniWrapper
import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects
import org.apache.spark.sql.execution.datasources.v1.clickhouse._
import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts}
+import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils}
@@ -264,12 +265,21 @@ object OptimizeTableCommandOverwrites extends Logging {
operationName: String,
hadoopConf: Map[String, String] = Map.empty): DeltaLog = {
val tablePath =
- if (tableIdentifier.nonEmpty && isDeltaTable(spark, tableIdentifier.get)) {
+ if (path.nonEmpty) {
+ new Path(path.get)
+ } else if (tableIdentifier.nonEmpty) {
val sessionCatalog = spark.sessionState.catalog
lazy val metadata = sessionCatalog.getTableMetadata(tableIdentifier.get)
- new Path(metadata.location)
+
+ if (CHDataSourceUtils.isClickhousePath(spark, tableIdentifier.get)) {
+ new Path(tableIdentifier.get.table)
+ } else if (CHDataSourceUtils.isClickHouseTable(spark, tableIdentifier.get)) {
+ new Path(metadata.location)
+ } else {
+ throw DeltaErrors.notADeltaTableException(operationName)
+ }
} else {
- throw new UnsupportedOperationException("OPTIMIZE is ony supported for clickhouse tables")
+ throw DeltaErrors.missingTableIdentifierException(operationName)
}
val startTime = Some(System.currentTimeMillis)
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala
index 4f96ef812..8c1062f4c 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala
@@ -72,7 +72,8 @@ class ClickHouseDataSource extends DeltaDataSource {
.getOrElse(Nil)
val deltaLog = DeltaLog.forTable(sqlContext.sparkSession, path, parameters)
- val configs = if (DeltaLogAdapter.snapshot(deltaLog).version < 0) {
+ // need to use the latest snapshot
+ val configs = if (deltaLog.update().version < 0) {
// when creating table, save the clickhouse config to the delta metadata
val clickHouseTableV2 = ClickHouseTableV2.getTable(deltaLog)
clickHouseTableV2.properties().asScala.toMap ++ DeltaConfigs
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala
index daaec00a7..c2d208df5 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala
@@ -227,7 +227,7 @@ class ClickHouseSparkCatalog
override def loadTable(ident: Identifier): Table = {
try {
super.loadTable(ident) match {
- case v1: V1Table if CHDataSourceUtils.isDeltaTable(v1.catalogTable) =>
+ case v1: V1Table if CHDataSourceUtils.isClickHouseTable(v1.catalogTable) =>
new ClickHouseTableV2(
spark,
new Path(v1.catalogTable.location),
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala
index 190315713..e2e7ac491 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala
@@ -16,8 +16,12 @@
*/
package org.apache.spark.sql.execution.datasources.v2.clickhouse.utils
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{AnalysisException, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError
import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig
import org.apache.hadoop.fs.Path
@@ -26,10 +30,11 @@ import java.util.Locale
import scala.util.Try
-object CHDataSourceUtils {
+object CHDataSourceUtils extends Logging {
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
/** Check whether this table is a Delta table based on information from the Catalog. */
- def isDeltaTable(table: CatalogTable): Boolean =
+ def isClickHouseTable(table: CatalogTable): Boolean =
CHDataSourceUtils.isClickHouseTable(table.provider)
/** Check whether this table is a ClickHouse table based on information from the Catalog. */
@@ -42,6 +47,48 @@ object CHDataSourceUtils {
name.toLowerCase(Locale.ROOT) == ClickHouseConfig.ALT_NAME
}
+ /**
+ * Check whether the provided table name is a Clickhouse table based on information from the
+ * Catalog.
+ */
+ def isClickHouseTable(spark: SparkSession, tableName: TableIdentifier): Boolean = {
+ val catalog = spark.sessionState.catalog
+ val tableIsNotTemporaryTable = !catalog.isTempView(tableName)
+ val tableExists = {
+ (tableName.database.isEmpty || catalog.databaseExists(tableName.database.get)) &&
+ catalog.tableExists(tableName)
+ }
+ tableIsNotTemporaryTable &&
+ tableExists &&
+ isClickHouseTable(catalog.getTableMetadata(tableName))
+ }
+
+ /** Check the specified table identifier represents a Clickhouse path. */
+ def isClickhousePath(spark: SparkSession, identifier: TableIdentifier): Boolean = {
+ val catalog = spark.sessionState.catalog
+
+ def tableIsTemporaryTable = catalog.isTempView(identifier)
+
+ def tableExists: Boolean = {
+ try {
+ catalog.databaseExists(identifier.database.get) && catalog.tableExists(identifier)
+ } catch {
+ case e: AnalysisException if gluePermissionError(e) =>
+ logWarning(
+ "Received an access denied error from Glue. Will check to see if this " +
+ s"identifier ($identifier) is path based.",
+ e)
+ false
+ }
+ }
+
+ spark.sessionState.conf.runSQLonFile &&
+ isClickHouseTable(identifier.database) &&
+ !tableIsTemporaryTable &&
+ !tableExists &&
+ new Path(identifier.table).isAbsolute
+ }
+
/** Find the root of a Delta table from the provided path. */
def findClickHouseTableRoot(
spark: SparkSession,
@@ -49,7 +96,10 @@ object CHDataSourceUtils {
options: Map[String, String] = Map.empty): Option[Path] = {
val fs = path.getFileSystem(spark.sessionState.newHadoopConfWithOptions(options))
var currentPath = path
- while (currentPath != null && currentPath.getName != ClickHouseConfig.METADATA_DIR) {
+ while (
+ currentPath != null && currentPath.getName != ClickHouseConfig.METADATA_DIR &&
+ currentPath.getName != "_samples"
+ ) {
val deltaLogPath = new Path(currentPath, ClickHouseConfig.METADATA_DIR)
if (Try(fs.exists(deltaLogPath)).getOrElse(false)) {
return Option(currentPath)
@@ -58,4 +108,24 @@ object CHDataSourceUtils {
}
None
}
+
+ // Ensure ClickHouseTableV2 table exists
+ def ensureClickHouseTableV2(
+ tableId: Option[TableIdentifier],
+ sparkSession: SparkSession): Unit = {
+ if (tableId.isEmpty) {
+ throw new UnsupportedOperationException("Current command requires table identifier.")
+ }
+ // If user comes into this function without previously triggering loadTable
+ // (which creates ClickhouseTableV2), we have to load the table manually
+ // Notice: Multi-catalog case is not well considered!
+ val table = sparkSession.sessionState.catalogManager.currentCatalog.asTableCatalog.loadTable(
+ Identifier.of(
+ Array(
+ tableId.get.database.getOrElse(
+ sparkSession.sessionState.catalogManager.currentNamespace.head)),
+ tableId.get.table)
+ )
+ table.name()
+ }
}
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
index c96058f55..9635b9958 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
@@ -20,6 +20,8 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import io.delta.tables.ClickhouseTable
+
import java.io.File
// Some sqls' line length exceeds 100
@@ -361,9 +363,8 @@ class GlutenClickHouseMergeTreeOptimizeSuite
}
}
- // TODO: support later
- ignore("test mergetree optimize with the path based table") {
- val dataPath = s"$basePath/lineitem_mergetree_optimize_p1"
+ test("test mergetree optimize with the path based table") {
+ val dataPath = s"$basePath/lineitem_mergetree_optimize_path_based"
clearDataPath(dataPath)
withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "838000") {
// 3 from 37 parts are larger than this, so after optimize there should be 4 parts:
@@ -378,15 +379,16 @@ class GlutenClickHouseMergeTreeOptimizeSuite
.mode(SaveMode.Append)
.save(dataPath)
- spark.sql(s"optimize clickhouse.`$dataPath`")
+ val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+ clickhouseTable.optimize().executeCompaction()
spark.sql("set spark.gluten.enabled=false")
- spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
- spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
+ clickhouseTable.vacuum(0.0)
+ clickhouseTable.vacuum(0.0)
spark.sql("set spark.gluten.enabled=true")
assert(countFiles(new File(dataPath)) == 99)
- val ret = spark.sql("select count(*) from clickhouse.`${dataPath}`").collect()
+ val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect()
assert(ret.apply(0).get(0) == 600572)
}
@@ -398,28 +400,30 @@ class GlutenClickHouseMergeTreeOptimizeSuite
// 1 merged part from 2 original parts, 1 merged part from 34 original parts
// and 1 original part (size 838255)
- spark.sql("optimize clickhouse.`${dataPath}`")
+ val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+ clickhouseTable.optimize().executeCompaction()
spark.sql("set spark.gluten.enabled=false")
- spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
- spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
+ clickhouseTable.vacuum(0.0)
+ clickhouseTable.vacuum(0.0)
spark.sql("set spark.gluten.enabled=true")
assert(countFiles(new File(dataPath)) == 93)
- val ret = spark.sql("select count(*) from clickhouse.`${dataPath}`").collect()
+ val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect()
assert(ret.apply(0).get(0) == 600572)
}
// now merge all parts (testing merging from merged parts)
- spark.sql("optimize clickhouse.`${dataPath}`")
+ val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+ clickhouseTable.optimize().executeCompaction()
spark.sql("set spark.gluten.enabled=false")
- spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
- spark.sql("VACUUM clickhouse.`${dataPath}` RETAIN 0 HOURS")
+ clickhouseTable.vacuum(0.0)
+ clickhouseTable.vacuum(0.0)
spark.sql("set spark.gluten.enabled=true")
assert(countFiles(new File(dataPath)) == 77)
- val ret = spark.sql("select count(*) from clickhouse.`${dataPath}`").collect()
+ val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect()
assert(ret.apply(0).get(0) == 600572)
}
}
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala
index dcaf69ee4..d85217536 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts
import org.apache.spark.sql.functions._
+import io.delta.tables.ClickhouseTable
import org.apache.commons.io.filefilter.WildcardFileFilter
import java.io.File
@@ -405,6 +406,37 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite
assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted))
}
+ val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+ clickhouseTable.updateExpr("l_orderkey = 10086", Map("l_returnflag" -> "'X'"))
+
+ {
+ val df = spark.read
+ .format("clickhouse")
+ .load(dataPath)
+ .where("l_returnflag = 'X'")
+ assert(df.count() == 1)
+ val scanExec = collect(df.queryExecution.executedPlan) {
+ case f: FileSourceScanExecTransformer => f
+ }
+ assert(scanExec.size == 1)
+
+ val mergetreeScan = scanExec.head
+ assert(mergetreeScan.nodeName.startsWith("Scan mergetree"))
+
+ val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex]
+ val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts])
+ assert(
+ addFiles.map(_.rows).sum
+ == 600572)
+
+ // 4 parts belong to the first batch
+ // 2 parts belong to the second batch (1 actual updated part, 1 passively updated).
+ assert(addFiles.size == 6)
+ val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_")))
+ assert(filePaths.size == 2)
+ assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted))
+ }
+
val df = spark.read
.format("clickhouse")
.load(dataPath)
@@ -444,9 +476,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite
assert(filePaths.size == 2)
assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted))
- spark.sql(s"""
- | delete from clickhouse.`$dataPath` where mod(l_orderkey, 3) = 2
- |""".stripMargin)
+ val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
+ clickhouseTable.delete("mod(l_orderkey, 3) = 2")
val df1 = spark.read
.format("clickhouse")
.load(dataPath)
@@ -1272,7 +1303,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite
checkQueryResult(dataPath)
- val dataPath1 = s"$basePath/lineitem_mergetree_ctas_5219_1"
+ val dataPath1 = s"$basePath/lineitem_mergetree_ctas_5219"
+ clearDataPath(dataPath1)
sourceDF.write
.format("clickhouse")
.mode(SaveMode.Append)
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala
index 7f507af23..49011e031 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala
@@ -535,7 +535,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite
}
ignore("test mergetree write with the path based") {
- val dataPath = s"$HDFS_URL/test/lineitem_mergetree_bucket_path_based_hdfs"
+ val dataPath = s"$HDFS_URL/test/lineitem_mergetree_bucket_hdfs"
val sourceDF = spark.sql(s"""
|select * from lineitem
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala
index 95df0a086..2b6414388 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala
@@ -552,7 +552,7 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite
}
test("test mergetree write with the path based") {
- val dataPath = s"s3a://$BUCKET_NAME/lineitem_mergetree_bucket_path_based_s3"
+ val dataPath = s"s3a://$BUCKET_NAME/lineitem_mergetree_bucket_s3"
val sourceDF = spark.sql(s"""
|select * from lineitem
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org