You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2025/04/24 15:48:48 UTC
(incubator-gluten) branch main updated: [VL] Remove unused logic in ColumnarCachedBatchSerializer#supportsColumnarInput (#9413)
This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7a1357514f [VL] Remove unused logic in ColumnarCachedBatchSerializer#supportsColumnarInput (#9413)
7a1357514f is described below
commit 7a1357514f26f2be74bd95601859aa11aa13ce5d
Author: Hongze Zhang <ho...@gmail.com>
AuthorDate: Thu Apr 24 16:48:45 2025 +0100
[VL] Remove unused logic in ColumnarCachedBatchSerializer#supportsColumnarInput (#9413)
---
.../spark/sql/execution/ColumnarCachedBatchSerializer.scala | 13 +------------
1 file changed, 1 insertion(+), 12 deletions(-)
diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
index e86d49ccef..1d52e57ab2 100644
--- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
+++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
@@ -101,18 +101,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with Logging {
}
override def supportsColumnarInput(schema: Seq[Attribute]): Boolean = {
- // Note, there is a issue that, if gluten columnar scan is disabled and vanilla Spark
- // columnar is enabled, then the following plan would fail.
- // InMemoryTableScan
- // InMemoryRelation
- // (vanilla Spark columnar Scan) Parquet
- // The reason is that, Spark will remove the top level `ColumnarToRow` and call
- // `convertColumnarBatchToCachedBatch`, but the inside ColumnarBatch is not arrow-based.
- // See: `InMemoryRelation.apply()`.
- // So we should disallow columnar input if using vanilla Spark columnar scan.
- val noVanillaSparkColumnarScan = glutenConf.enableColumnarFileScan ||
- !glutenConf.getConf(GlutenConfig.VANILLA_VECTORIZED_READERS_ENABLED)
- glutenConf.enableGluten && validateSchema(schema) && noVanillaSparkColumnarScan
+ glutenConf.enableGluten && validateSchema(schema)
}
override def supportsColumnarOutput(schema: StructType): Boolean = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org