You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2024/12/17 01:37:55 UTC
(incubator-gluten) branch main updated: [GLUTEN-7911][CORE] Flip dependency direction for gluten-hudi (#8240)
This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 82b48b38d1 [GLUTEN-7911][CORE] Flip dependency direction for gluten-hudi (#8240)
82b48b38d1 is described below
commit 82b48b38d10217ea4921248d3e4ffc5fc48877b7
Author: Hongze Zhang <ho...@intel.com>
AuthorDate: Tue Dec 17 09:37:51 2024 +0800
[GLUTEN-7911][CORE] Flip dependency direction for gluten-hudi (#8240)
Closes #7911
---
backends-velox/pom.xml | 23 ++++++++++++++++++++++
.../org/apache/execution/VeloxHudiSuite.scala | 21 ++++++++++++++++++++
.../org/apache}/execution/VeloxTPCHHudiSuite.scala | 8 ++++----
.../memory/memtarget/spark/TreeMemoryConsumer.java | 3 ++-
.../org/apache/spark/task/TaskResources.scala | 11 ++++++++---
gluten-hudi/pom.xml | 13 ------------
.../{VeloxHudiSuite.scala => HudiSuite.scala} | 2 +-
.../apache/gluten/utils/BackendTestSettings.scala | 3 ++-
.../sql/execution/AbstractFileSourceScanExec.scala | 4 +++-
.../sql/execution/AbstractFileSourceScanExec.scala | 4 +++-
.../sql/execution/AbstractFileSourceScanExec.scala | 4 +++-
.../sql/execution/AbstractFileSourceScanExec.scala | 4 +++-
12 files changed, 73 insertions(+), 27 deletions(-)
diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
index ed0bf20616..9349c3c092 100755
--- a/backends-velox/pom.xml
+++ b/backends-velox/pom.xml
@@ -111,6 +111,29 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>hudi</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-hudi</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-hudi</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
+ <version>${hudi.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
<dependencies>
diff --git a/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxHudiSuite.scala b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxHudiSuite.scala
new file mode 100644
index 0000000000..00498f8741
--- /dev/null
+++ b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxHudiSuite.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.execution
+
+import org.apache.gluten.execution.HudiSuite
+
+class VeloxHudiSuite extends HudiSuite {}
diff --git a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxTPCHHudiSuite.scala
similarity index 91%
rename from gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala
rename to backends-velox/src-hudi/test/scala/org/apache/execution/VeloxTPCHHudiSuite.scala
index a4e10269c2..cdb3b29180 100644
--- a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala
+++ b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxTPCHHudiSuite.scala
@@ -14,16 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.execution
+package org.apache.execution
+import org.apache.gluten.execution.VeloxTPCHSuite
import org.apache.spark.SparkConf
import java.io.File
class VeloxTPCHHudiSuite extends VeloxTPCHSuite {
-
- protected val tpchBasePath: String = new File(
- "../backends-velox/src/test/resources").getAbsolutePath
+ protected val tpchBasePath: String =
+ getClass.getResource("/").getPath + "../../../src/test/resources"
override protected val resourcePath: String =
new File(tpchBasePath, "tpch-data-parquet").getCanonicalPath
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
index 44c725798c..1289a01c34 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
@@ -138,7 +138,8 @@ public class TreeMemoryConsumer extends MemoryConsumer implements TreeMemoryTarg
@Override
public TreeMemoryTarget parent() {
// we are root
- throw new IllegalStateException("Unreachable code");
+ throw new IllegalStateException(
+ "Unreachable code org.apache.gluten.memory.memtarget.spark.TreeMemoryConsumer.parent");
}
@Override
diff --git a/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
index b061aa332c..df5917125b 100644
--- a/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
@@ -298,9 +298,14 @@ class TaskResourceRegistry extends Logging {
o1: util.Map.Entry[Int, util.LinkedHashSet[TaskResource]],
o2: util.Map.Entry[Int, util.LinkedHashSet[TaskResource]]) => {
val diff = o2.getKey - o1.getKey // descending by priority
- if (diff > 0) 1
- else if (diff < 0) -1
- else throw new IllegalStateException("Unreachable code")
+ if (diff > 0) {
+ 1
+ } else if (diff < 0) {
+ -1
+ } else {
+ throw new IllegalStateException(
+ "Unreachable code from org.apache.spark.task.TaskResourceRegistry.releaseAll")
+ }
}
)
table.forEach {
diff --git a/gluten-hudi/pom.xml b/gluten-hudi/pom.xml
index 7900182f85..5865f1f6ec 100755
--- a/gluten-hudi/pom.xml
+++ b/gluten-hudi/pom.xml
@@ -46,19 +46,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.gluten</groupId>
- <artifactId>backends-velox</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.gluten</groupId>
- <artifactId>backends-velox</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
diff --git a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/HudiSuite.scala
similarity index 98%
rename from gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala
rename to gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/HudiSuite.scala
index b760ec5565..97633fa064 100644
--- a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala
+++ b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/HudiSuite.scala
@@ -19,7 +19,7 @@ package org.apache.gluten.execution
import org.apache.spark.SparkConf
import org.apache.spark.sql.Row
-class VeloxHudiSuite extends WholeStageTransformerSuite {
+abstract class HudiSuite extends WholeStageTransformerSuite {
protected val rootPath: String = getClass.getResource("/").getPath
override protected val resourcePath: String = "/tpch-data-parquet"
diff --git a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala
index dce8ac8371..51e8174da7 100644
--- a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala
+++ b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala
@@ -80,7 +80,8 @@ abstract class BackendTestSettings {
return !isExcluded
}
- throw new IllegalStateException("Unreachable code")
+ throw new IllegalStateException(
+ "Unreachable code from org.apache.gluten.utils.BackendTestSettings.shouldRun")
}
final protected class SuiteSettings {
diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index a3bd5079b0..fcdd3c3c8b 100644
--- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -73,7 +73,9 @@ abstract class AbstractFileSourceScanExec(
override def supportsColumnar: Boolean = {
// The value should be defined in GlutenPlan.
- throw new UnsupportedOperationException("Unreachable code")
+ throw new UnsupportedOperationException(
+ "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+ ".supportsColumnar")
}
private lazy val needsUnsafeRowConversion: Boolean = {
diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index c885f0cf44..01df5ba621 100644
--- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -77,7 +77,9 @@ abstract class AbstractFileSourceScanExec(
override def supportsColumnar: Boolean = {
// The value should be defined in GlutenPlan.
- throw new UnsupportedOperationException("Unreachable code")
+ throw new UnsupportedOperationException(
+ "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+ ".supportsColumnar")
}
private lazy val needsUnsafeRowConversion: Boolean = {
diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index 53ea6f543a..15e54ddb71 100644
--- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -69,7 +69,9 @@ abstract class AbstractFileSourceScanExec(
override def supportsColumnar: Boolean = {
// The value should be defined in GlutenPlan.
- throw new UnsupportedOperationException("Unreachable code")
+ throw new UnsupportedOperationException(
+ "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+ ".supportsColumnar")
}
private lazy val needsUnsafeRowConversion: Boolean = {
diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index c8dbcc2fed..a83c763c45 100644
--- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -69,7 +69,9 @@ abstract class AbstractFileSourceScanExec(
override def supportsColumnar: Boolean = {
// The value should be defined in GlutenPlan.
- throw new UnsupportedOperationException("Unreachable code")
+ throw new UnsupportedOperationException(
+ "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+ ".supportsColumnar")
}
private lazy val needsUnsafeRowConversion: Boolean = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org