You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2024/12/17 01:37:55 UTC

(incubator-gluten) branch main updated: [GLUTEN-7911][CORE] Flip dependency direction for gluten-hudi (#8240)

This is an automated email from the ASF dual-hosted git repository.

hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 82b48b38d1 [GLUTEN-7911][CORE] Flip dependency direction for gluten-hudi (#8240)
82b48b38d1 is described below

commit 82b48b38d10217ea4921248d3e4ffc5fc48877b7
Author: Hongze Zhang <ho...@intel.com>
AuthorDate: Tue Dec 17 09:37:51 2024 +0800

    [GLUTEN-7911][CORE] Flip dependency direction for gluten-hudi (#8240)
    
    Closes #7911
---
 backends-velox/pom.xml                             | 23 ++++++++++++++++++++++
 .../org/apache/execution/VeloxHudiSuite.scala      | 21 ++++++++++++++++++++
 .../org/apache}/execution/VeloxTPCHHudiSuite.scala |  8 ++++----
 .../memory/memtarget/spark/TreeMemoryConsumer.java |  3 ++-
 .../org/apache/spark/task/TaskResources.scala      | 11 ++++++++---
 gluten-hudi/pom.xml                                | 13 ------------
 .../{VeloxHudiSuite.scala => HudiSuite.scala}      |  2 +-
 .../apache/gluten/utils/BackendTestSettings.scala  |  3 ++-
 .../sql/execution/AbstractFileSourceScanExec.scala |  4 +++-
 .../sql/execution/AbstractFileSourceScanExec.scala |  4 +++-
 .../sql/execution/AbstractFileSourceScanExec.scala |  4 +++-
 .../sql/execution/AbstractFileSourceScanExec.scala |  4 +++-
 12 files changed, 73 insertions(+), 27 deletions(-)

diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
index ed0bf20616..9349c3c092 100755
--- a/backends-velox/pom.xml
+++ b/backends-velox/pom.xml
@@ -111,6 +111,29 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>hudi</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.gluten</groupId>
+          <artifactId>gluten-hudi</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.gluten</groupId>
+          <artifactId>gluten-hudi</artifactId>
+          <version>${project.version}</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hudi</groupId>
+          <artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
+          <version>${hudi.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
   </profiles>
 
   <dependencies>
diff --git a/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxHudiSuite.scala b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxHudiSuite.scala
new file mode 100644
index 0000000000..00498f8741
--- /dev/null
+++ b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxHudiSuite.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.execution
+
+import org.apache.gluten.execution.HudiSuite
+
+class VeloxHudiSuite extends HudiSuite {}
diff --git a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxTPCHHudiSuite.scala
similarity index 91%
rename from gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala
rename to backends-velox/src-hudi/test/scala/org/apache/execution/VeloxTPCHHudiSuite.scala
index a4e10269c2..cdb3b29180 100644
--- a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala
+++ b/backends-velox/src-hudi/test/scala/org/apache/execution/VeloxTPCHHudiSuite.scala
@@ -14,16 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.execution
+package org.apache.execution
 
+import org.apache.gluten.execution.VeloxTPCHSuite
 import org.apache.spark.SparkConf
 
 import java.io.File
 
 class VeloxTPCHHudiSuite extends VeloxTPCHSuite {
-
-  protected val tpchBasePath: String = new File(
-    "../backends-velox/src/test/resources").getAbsolutePath
+  protected val tpchBasePath: String =
+    getClass.getResource("/").getPath + "../../../src/test/resources"
 
   override protected val resourcePath: String =
     new File(tpchBasePath, "tpch-data-parquet").getCanonicalPath
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
index 44c725798c..1289a01c34 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
@@ -138,7 +138,8 @@ public class TreeMemoryConsumer extends MemoryConsumer implements TreeMemoryTarg
   @Override
   public TreeMemoryTarget parent() {
     // we are root
-    throw new IllegalStateException("Unreachable code");
+    throw new IllegalStateException(
+        "Unreachable code org.apache.gluten.memory.memtarget.spark.TreeMemoryConsumer.parent");
   }
 
   @Override
diff --git a/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
index b061aa332c..df5917125b 100644
--- a/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
@@ -298,9 +298,14 @@ class TaskResourceRegistry extends Logging {
           o1: util.Map.Entry[Int, util.LinkedHashSet[TaskResource]],
           o2: util.Map.Entry[Int, util.LinkedHashSet[TaskResource]]) => {
         val diff = o2.getKey - o1.getKey // descending by priority
-        if (diff > 0) 1
-        else if (diff < 0) -1
-        else throw new IllegalStateException("Unreachable code")
+        if (diff > 0) {
+          1
+        } else if (diff < 0) {
+          -1
+        } else {
+          throw new IllegalStateException(
+            "Unreachable code from org.apache.spark.task.TaskResourceRegistry.releaseAll")
+        }
       }
     )
     table.forEach {
diff --git a/gluten-hudi/pom.xml b/gluten-hudi/pom.xml
index 7900182f85..5865f1f6ec 100755
--- a/gluten-hudi/pom.xml
+++ b/gluten-hudi/pom.xml
@@ -46,19 +46,6 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.gluten</groupId>
-      <artifactId>backends-velox</artifactId>
-      <version>${project.version}</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.gluten</groupId>
-      <artifactId>backends-velox</artifactId>
-      <version>${project.version}</version>
-      <type>test-jar</type>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
diff --git a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/HudiSuite.scala
similarity index 98%
rename from gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala
rename to gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/HudiSuite.scala
index b760ec5565..97633fa064 100644
--- a/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala
+++ b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/HudiSuite.scala
@@ -19,7 +19,7 @@ package org.apache.gluten.execution
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.Row
 
-class VeloxHudiSuite extends WholeStageTransformerSuite {
+abstract class HudiSuite extends WholeStageTransformerSuite {
 
   protected val rootPath: String = getClass.getResource("/").getPath
   override protected val resourcePath: String = "/tpch-data-parquet"
diff --git a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala
index dce8ac8371..51e8174da7 100644
--- a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala
+++ b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala
@@ -80,7 +80,8 @@ abstract class BackendTestSettings {
       return !isExcluded
     }
 
-    throw new IllegalStateException("Unreachable code")
+    throw new IllegalStateException(
+      "Unreachable code from org.apache.gluten.utils.BackendTestSettings.shouldRun")
   }
 
   final protected class SuiteSettings {
diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index a3bd5079b0..fcdd3c3c8b 100644
--- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -73,7 +73,9 @@ abstract class AbstractFileSourceScanExec(
 
   override def supportsColumnar: Boolean = {
     // The value should be defined in GlutenPlan.
-    throw new UnsupportedOperationException("Unreachable code")
+    throw new UnsupportedOperationException(
+      "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+        ".supportsColumnar")
   }
 
   private lazy val needsUnsafeRowConversion: Boolean = {
diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index c885f0cf44..01df5ba621 100644
--- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -77,7 +77,9 @@ abstract class AbstractFileSourceScanExec(
 
   override def supportsColumnar: Boolean = {
     // The value should be defined in GlutenPlan.
-    throw new UnsupportedOperationException("Unreachable code")
+    throw new UnsupportedOperationException(
+      "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+        ".supportsColumnar")
   }
 
   private lazy val needsUnsafeRowConversion: Boolean = {
diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index 53ea6f543a..15e54ddb71 100644
--- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -69,7 +69,9 @@ abstract class AbstractFileSourceScanExec(
 
   override def supportsColumnar: Boolean = {
     // The value should be defined in GlutenPlan.
-    throw new UnsupportedOperationException("Unreachable code")
+    throw new UnsupportedOperationException(
+      "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+        ".supportsColumnar")
   }
 
   private lazy val needsUnsafeRowConversion: Boolean = {
diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
index c8dbcc2fed..a83c763c45 100644
--- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
+++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala
@@ -69,7 +69,9 @@ abstract class AbstractFileSourceScanExec(
 
   override def supportsColumnar: Boolean = {
     // The value should be defined in GlutenPlan.
-    throw new UnsupportedOperationException("Unreachable code")
+    throw new UnsupportedOperationException(
+      "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" +
+        ".supportsColumnar")
   }
 
   private lazy val needsUnsafeRowConversion: Boolean = {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org