You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by fe...@apache.org on 2025/01/23 20:41:39 UTC
(incubator-gluten) branch main updated: [GLUTEN-7544][CORE] Add Qualification Tool (#8484)
This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a1e7258577 [GLUTEN-7544][CORE] Add Qualification Tool (#8484)
a1e7258577 is described below
commit a1e725857723bd798d74d49ee47351b6e38ede35
Author: Srinivas S T <40...@users.noreply.github.com>
AuthorDate: Fri Jan 24 02:11:34 2025 +0530
[GLUTEN-7544][CORE] Add Qualification Tool (#8484)
Adding a qualification tool that can go through Spark event files and output percentage of the application that can be run using Gluten.
---
README.md | 25 ++-
pom.xml | 10 +
tools/qualification-tool/.gitignore | 1 +
tools/qualification-tool/README.MD | 217 ++++++++++++++++++++
tools/qualification-tool/pom.xml | 208 +++++++++++++++++++
.../org/apache/gluten/qt/graph/MetricInternal.java | 41 ++++
.../qt/graph/SparkPlanGraphClusterInternal.java | 38 ++++
.../qt/graph/SparkPlanGraphEdgeInternal.java | 36 ++++
.../gluten/qt/graph/SparkPlanGraphInternal.java | 47 +++++
.../qt/graph/SparkPlanGraphNodeInternal.java | 69 +++++++
.../gluten/qt/graph/SqlPlanMetricInternal.java | 42 ++++
.../gluten/qt/support/ChildSupportVisitor.java | 69 +++++++
.../gluten/qt/support/ClusterSupportVisitor.java | 88 ++++++++
.../apache/gluten/qt/support/DurationMetric.java | 83 ++++++++
.../gluten/qt/support/ExecutionDescription.java | 148 ++++++++++++++
.../qt/support/GlutenExecutionSupportReporter.java | 59 ++++++
.../apache/gluten/qt/support/GlutenSupport.java | 24 +++
.../org/apache/gluten/qt/support/GraphVisitor.java | 59 ++++++
.../gluten/qt/support/NodeSupportVisitor.java | 225 ++++++++++++++++++++
.../org/apache/gluten/qt/support/NotSupported.java | 57 ++++++
.../gluten/qt/support/NotSupportedCategory.java | 25 +++
.../apache/gluten/qt/support/ResultVisitor.java | 182 +++++++++++++++++
.../org/apache/gluten/qt/support/Supported.java | 31 +++
.../org/apache/gluten/qt/AppStatusListener.scala | 37 ++++
.../org/apache/gluten/qt/AppStatusStore.scala | 73 +++++++
.../org/apache/gluten/qt/QualificationTool.scala | 95 +++++++++
.../gluten/qt/QualificationToolConfiguration.scala | 91 +++++++++
.../org/apache/gluten/qt/ReflectionUtils.scala | 60 ++++++
.../org/apache/gluten/qt/ReplayListenerBus.scala | 51 +++++
.../apache/gluten/qt/SQLAppStatusListener.scala | 35 ++++
.../scala/org/apache/gluten/qt/SparkListener.scala | 21 ++
.../org/apache/gluten/qt/execution/Executor.scala | 226 +++++++++++++++++++++
.../org/apache/gluten/qt/file/GcsFileSource.scala | 45 ++++
.../apache/gluten/qt/file/HadoopFileSource.scala | 89 ++++++++
.../apache/gluten/qt/file/LocalFsFileSource.scala | 38 ++++
.../support/GlutenApplicationSupportReporter.scala | 155 ++++++++++++++
.../apache/gluten/qt/support/SupportReport.scala | 27 +++
.../gluten/qt/writer/ApplicationReport.scala | 48 +++++
.../org/apache/gluten/qt/writer/ImpactReport.scala | 27 +++
.../qt/writer/OperatorImpactReportWriter.scala | 45 ++++
.../gluten/qt/writer/PrimaryReportWriter.scala | 45 ++++
.../scala/org/apache/gluten/qt/writer/Report.scala | 21 ++
.../org/apache/gluten/qt/writer/ReportWriter.scala | 82 ++++++++
.../org/apache/gluten/qt/writer/WriteUtils.scala | 48 +++++
.../apache/gluten/qt/QualificationToolTest.java | 56 +++++
.../src/test/resources/AppsRecommendedForBoost.tsv | 4 +
.../src/test/resources/UnsupportedOperators.tsv | 3 +
.../src/test/resources/event-files/local-rdd | 13 ++
.../test/resources/event-files/local-sql-supported | 18 ++
.../test/resources/event-files/local-unsupported | 24 +++
50 files changed, 3259 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index 74b51af20e..831d8df36e 100644
--- a/README.md
+++ b/README.md
@@ -123,10 +123,31 @@ The below testing environment: a 8-nodes AWS cluster with 1TB data; Spark-3.1.1
-# 8. License
+# 8. Qualification Tool
+
+The Qualification Tool is a utility to analyze Spark event log files and assess the compatibility and performance of SQL workloads with Gluten. This tool helps users understand how their workloads can benefit from Gluten.
+
+## Features
+- Analyzes Spark SQL execution plans for compatibility with Gluten.
+- Supports various types of event log files, including single files, folders, compressed files, and rolling event logs.
+- Generates detailed reports highlighting supported and unsupported operations.
+- Provides metrics on SQL execution times and operator impact.
+- Offers configurable options such as threading, output directory, and date-based filtering.
+
+## Usage
+
+To use the Qualification Tool, follow the instructions in its [README](tools/qualification-tool/README.MD).
+
+## Example Command
+```bash
+java -jar target/qualification-tool-1.3.0-SNAPSHOT-jar-with-dependencies.jar -f /path/to/eventlog
+```
+For detailed usage instructions and advanced options, see the Qualification Tool README.
+
+# 9. License
Gluten is licensed under [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0).
-# 9. Acknowledgements
+# 10. Acknowledgements
Gluten was initiated by Intel and Kyligence in 2022. Several companies are also actively participating in the development, such as BIGO, Meituan, Alibaba Cloud, NetEase, Baidu, Microsoft, IBM, Google, etc.
<a href="https://github.com/apache/incubator-gluten/graphs/contributors">
diff --git a/pom.xml b/pom.xml
index 4000197a2b..07cd0d5f3a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -401,6 +401,16 @@
</properties>
</profile>
+ <profile>
+ <id>qualification-tool</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <modules>
+ <module>tools/qualification-tool</module>
+ </modules>
+ </profile>
+
<profile>
<id>backends-clickhouse</id>
<activation>
diff --git a/tools/qualification-tool/.gitignore b/tools/qualification-tool/.gitignore
new file mode 100644
index 0000000000..1de565933b
--- /dev/null
+++ b/tools/qualification-tool/.gitignore
@@ -0,0 +1 @@
+target
\ No newline at end of file
diff --git a/tools/qualification-tool/README.MD b/tools/qualification-tool/README.MD
new file mode 100644
index 0000000000..157da5293b
--- /dev/null
+++ b/tools/qualification-tool/README.MD
@@ -0,0 +1,217 @@
+# Qualification Tool
+
+The Qualification Tool analyzes Spark event log files to determine the compatibility and performance of SQL workloads with Gluten.
+
+## Build
+
+To compile and package the Qualification Tool, run the following Maven command:
+
+```bash
+mvn clean package
+```
+
+This will create a jar file in the `target` directory.
+
+## Run
+
+To execute the tool, use the following command:
+
+```bash
+java -jar target/qualification-tool-1.3.0-SNAPSHOT-jar-with-dependencies.jar -f <eventFile>
+```
+
+### Parameters:
+- **`-f <eventFile>`**: Path to the Spark event log file(s). This can be:
+ - A single event log file.
+ - A folder containing multiple event log files.
+ - Deeply nested folders of event log files.
+ - Compressed event log files.
+ - Rolling event log files.
+ - Comma separated files
+- **`-k <gcsKey>`**: (Optional) Path to Google Cloud Storage service account keys.
+- **`-o <output>`**: (Optional) Path to the directory where output will be written. Defaults to a temporary directory.
+- **`-t <threads>`**: (Optional) Number of processing threads. Defaults to 4.
+- **`-v`**: (Optional) Enable non verbose output. Omit this flag for verbose mode.
+- **`-p <project>`**: (Optional) Project ID for the run.
+- **`-d <dateFilter>`**: (Optional) Analyze only files created after this date (format: YYYY-MM-DD). Defaults to the last 90 days.
+
+### Example Usage:
+```bash
+java -jar target/qualification-tool-1.3.0-SNAPSHOT-jar-with-dependencies.jar -f /path/to/eventlog
+```
+
+### Advanced Example:
+```bash
+java -jar target/qualification-tool-1.3.0-SNAPSHOT-jar-with-dependencies.jar -f /path/to/folder -o /output/path -t 8 -d 2023-01-01 -k /path/to/gcs_keys.json -p my_project
+```
+
+## Features
+
+- Analyzes Spark SQL execution plans for compatibility with Gluten.
+- Supports single files, folders, deeply nested folders, compressed files, and rolling event logs.
+- Provides detailed reports on supported and unsupported operations.
+- Generates metrics on SQL execution times and operator impact.
+- Configurable verbosity and threading.
+
+## How It Works
+
+The Qualification Tool analyzes a Spark plan to determine the compatibility of its nodes / operators and clusters with Gluten. Here's a step-by-step explanation of the process:
+
+### Example Spark Plan
+
+Consider the following Spark plan:
+
+```
+ G
+ / \
+ G[2] G[3]
+ | |
+ S[2] G[3]
+ | |
+ G S
+ / \
+ G[1] G
+ |
+ G[1]
+ |
+ G
+```
+
+- **G**: Represents a plan supported by Gluten.
+- **S**: Represents a plan not supported by Gluten.
+- **[1], [2], [3]**: Indicates the node belongs to a Whole Stage Code Gen Block (Cluster).
+
+### 1. NodeSupportVisitor
+
+The first step is marking each node as supported (`*`) or not supported (`!`) by Gluten:
+
+```
+ *G
+ / \
+ *G[2] *G[3]
+ | |
+ !S[2] *G[3]
+ | |
+ *G !S
+ / \
+ *G[1] *G
+ |
+ *G[1]
+ |
+ *G
+```
+
+- All supported nodes are marked with `*`.
+- All unsupported nodes are marked with `!`.
+
+### 2. ClusterSupportVisitor
+
+The second step marks entire clusters as not supported (`!`) if any node in the cluster is unsupported:
+
+```
+ *G
+ / \
+ !G[2] *G[3]
+ | |
+ !S[2] *G[3]
+ | |
+ *G !S
+ / \
+ *G[1] *G
+ |
+ *G[1]
+ |
+ *G
+```
+
+#### Reasoning:
+Although Gluten supports these operators, breaking Whole Stage Code Gen (WSCG) boundaries introduces row-to-columnar and columnar-to-row conversions, degrading performance. Hence, we pessimistically mark the entire cluster as not supported.
+
+### 3. ChildSupportVisitor
+
+The final step marks nodes and their parents as not supported if their children are unsupported:
+
+```
+ !G
+ / \
+ !G[2] !G[3]
+ | |
+ !S[2] !G[3]
+ | |
+ *G !S
+ / \
+ *G[1] *G
+ |
+ *G[1]
+ |
+ *G
+```
+
+#### Reasoning:
+If a child node is not supported by Gluten, row-to-columnar and columnar-to-row conversions are added, degrading performance. Therefore, we pessimistically mark such nodes as not supported.
+
+### Sample Output
+#### AppsRecommendedForBoost.tsv
+| applicationId | applicationName | batchUuid | rddPercentage | unsupportedSqlPercentage | totalTaskTime | supportedTaskTime | supportedSqlPercentage | recommendedForBoost | expectedRuntimeReduction |
+|-----------------------------------|-----------------------------------|-----------------------------------------|---------------|--------------------------|---------------|-------------------|-------------------------|---------------------|--------------------------|
+| app-20241001064609-0000 | POC - StoreItem | a56bee42 | 0.0% | 5.2% | 3244484900 | 3074633672 | 94.7% | true | 28.4% |
+| application_1729530422325_0001 | job-audience-generation-21 | UNKNOWN | 0.3% | 0.6% | 1378942742 | 1365259621 | 99.0% | true | 29.7% |
+| application_1729410290443_0001 | job-audience-generation-27 | UNKNOWN | 0.0% | 0.7% | 1212612881 | 1203690936 | 99.2% | true | 29.7% |
+| app-20241001080158-0000 | POC - StoreItem | 94088807 | 0.0% | 12.5% | 668601513 | 584812056 | 87.4% | true | 26.2% |
+| application_1729991008434_0001 | DailyLoad | UNKNOWN | 0.0% | 12.6% | 17134675 | 14963535 | 87.3% | true | 26.1% |
+| application_1730097715348_0003 | Spark shell | UNKNOWN | 0.0% | 0.0% | 4680 | 4680 | 100.0% | true | 30.0% |
+| application_1728526688221_0001 | job-audience-generation-27 | UNKNOWN | 0.4% | 59.4% | 805991113 | 323274568 | 40.1% | false | 12.0% |
+| application_1726070403340_0450 | driver.py | UNKNOWN | 0.0% | 81.1% | 398992332 | 75187879 | 18.8% | false | 5.6% |
+| application_1727842554841_0001 | driver.py | UNKNOWN | 0.0% | 58.3% | 166686890 | 69492539 | 41.6% | false | 12.5% |
+| application_1726070403340_0474 | driver.py | UNKNOWN | 0.0% | 81.6% | 325389669 | 59687634 | 18.3% | false | 5.5% |
+| application_1729133595704_0001 | job-audience-generation-54 | UNKNOWN | 0.6% | 99.3% | 472621872 | 20205 | 0.0% | false | 0.0% |
+| application_1730097715348_0002 | Spark shell | UNKNOWN | 33.8% | 16.6% | 4197 | 2077 | 49.5% | false | 14.8% |
+| application_1730097715348_0001 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1712155629437_0011 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1712155629437_0012 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1712155629437_0007 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| app-20241120163343-0000 | Config Test | c2884285 | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1712155629437_0008 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1712155629437_0010 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1730097715348_0004 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1718075857669_0261 | UnregisteredAsset | UNKNOWN | 0.0% | 100.0% | 12205458 | 0 | 0.0% | false | 0.0% |
+| application_1712155629437_0009 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+| application_1712155629437_0013 | Spark shell | UNKNOWN | 0.0% | 0.0% | 0 | 0 | 0.0% | false | 0.0% |
+
+#### UnsupportedOperators.tsv
+| unsupportedOperator | cumulativeCpuMs | count |
+|--------------------------------------------------------|-----------------|---------|
+| Scan with format JSON not supported | 1722606716687 | 3105 |
+| Parquet with struct | 1722606716687 | 3105 |
+| Execute InsertIntoHadoopFsRelationCommand not supported| 37884897400 | 405732 |
+| BHJ Not Supported | 20120117622 | 132164 |
+| SerializeFromObject not supported | 13856742899 | 16098 |
+| MapPartitions not supported | 13856742899 | 16098 |
+| FlatMapGroupsInPandas not supported | 257970400 | 224 |
+| Sample not supported | 157615298 | 432 |
+| DeserializeToObject not supported | 157610379 | 6721 |
+| MapElements not supported | 5261628 | 694 |
+| Parquet with struct and map | 29020 | 44 |
+| Execute CreateViewCommand not supported | 0 | 157 |
+| Execute CreateTableCommand not supported | 0 | 42 |
+| WriteFiles not supported | 0 | 20 |
+| Execute SetCommand not supported | 0 | 12 |
+| Execute SaveIntoDataSourceCommand not supported | 0 | 10 |
+| CreateTable not supported | 0 | 4 |
+| Execute RefreshTableCommand not supported | 0 | 3 |
+| Execute DeltaDynamicPartitionOverwriteCommand not supported | 0 | 2 |
+| Execute TruncateTableCommand not supported | 0 | 2 |
+| Execute AlterTableAddPartitionCommand not supported | 0 | 1 |
+| CreateNamespace not supported | 0 | 1 |
+
+
+### Summary
+
+The tool ensures that the Spark plan optimizes performance by:
+1. Identifying individual node compatibility.
+2. Accounting for cluster boundaries and WSCG optimizations.
+3. Considering child dependencies and their impact on parent nodes.
+
+## Requirements
+
+- **Java**: Ensure you have JDK 11 or later installed.
diff --git a/tools/qualification-tool/pom.xml b/tools/qualification-tool/pom.xml
new file mode 100644
index 0000000000..0b49fc723f
--- /dev/null
+++ b/tools/qualification-tool/pom.xml
@@ -0,0 +1,208 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>qualification-tool</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <name>Qualification Tool</name>
+
+ <properties>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ <encoding>UTF-8</encoding>
+ <scala.version>2.12.15</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
+ <spec2.version>4.2.0</spec2.version>
+ <spotless.version>2.27.2</spotless.version>
+ <spotless.scalafmt.version>3.8.3</spotless.scalafmt.version>
+ <spotless.delimiter>package</spotless.delimiter>
+ <spotless.license.header>
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ </spotless.license.header>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.12</artifactId>
+ <version>3.5.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ <version>hadoop2-2.2.24</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jspecify</groupId>
+ <artifactId>jspecify</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <sourceDir>src/main/scala</sourceDir>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.1</version>
+ <executions>
+ <execution>
+ <id>default-compile</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <configuration>
+ <source>11</source>
+ <target>11</target>
+ </configuration>
+ </execution>
+ <execution>
+ <id>default-testCompile</id>
+ <phase>test-compile</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.gluten.qt.QualificationTool</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for an identifier -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.4.2</version>
+ <configuration>
+ <archive>
+ <manifest>
+ <addClasspath>true</addClasspath>
+ <mainClass>org.apache.gluten.qt.QualificationTool</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.21.0</version>
+ </plugin>
+
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <version>${spotless.version}</version>
+ <configuration>
+ <java>
+ <toggleOffOn />
+ <googleJavaFormat>
+ <version>1.7</version>
+ </googleJavaFormat>
+
+ <!-- \# refers to the static imports -->
+ <importOrder>
+ <order>org.apache.gluten,io.substrait.spark,,javax,java,scala,\#</order>
+ </importOrder>
+
+ <removeUnusedImports />
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </java>
+ <scala>
+ <!-- make it works `// spotless:off ` -->
+ <toggleOffOn />
+ <scalafmt>
+ <version>${spotless.scalafmt.version}</version>
+ <scalaMajorVersion>${scala.binary.version}</scalaMajorVersion>
+ <file>../../.scalafmt.conf</file>
+ </scalafmt>
+ <licenseHeader>
+ <content>${spotless.license.header}</content>
+ <delimiter>${spotless.delimiter}</delimiter>
+ </licenseHeader>
+ </scala>
+ </configuration>
+ <executions>
+ <execution>
+ <id>spotless-check</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/MetricInternal.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/MetricInternal.java
new file mode 100644
index 0000000000..c25483fc26
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/MetricInternal.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.graph;
+
+/**
+ * An internal representation of a Spark execution metric.
+ *
+ * <p>Spark stores these metrics in a {@code Map<Long, String>}, and this class encapsulates each
+ * metric with its numeric key and string value.
+ */
+public class MetricInternal {
+ private long key;
+ private String metricValue;
+
+ public MetricInternal(long key, String metricValue) {
+ this.key = key;
+ this.metricValue = metricValue;
+ }
+
+ public long getKey() {
+ return key;
+ }
+
+ public String getMetricValue() {
+ return metricValue;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphClusterInternal.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphClusterInternal.java
new file mode 100644
index 0000000000..a2dc3ddb14
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphClusterInternal.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.graph;
+
+import java.util.List;
+
+/** Internal representation of a {@code org.apache.spark.sql.execution.ui.SparkPlanGraphCluster}. */
+public class SparkPlanGraphClusterInternal extends SparkPlanGraphNodeInternal {
+ private final List<SparkPlanGraphNodeInternal> nodes;
+
+ public SparkPlanGraphClusterInternal(
+ long id,
+ String name,
+ String desc,
+ List<SparkPlanGraphNodeInternal> nodes,
+ List<SqlPlanMetricInternal> metrics) {
+ super(id, name, desc, metrics);
+ this.nodes = nodes;
+ }
+
+ public List<SparkPlanGraphNodeInternal> getNodes() {
+ return nodes;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphEdgeInternal.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphEdgeInternal.java
new file mode 100644
index 0000000000..5a6f96d35c
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphEdgeInternal.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.graph;
+
+/** Internal representation of a {@code org.apache.spark.sql.execution.ui.SparkPlanGraphEdge}. */
+public class SparkPlanGraphEdgeInternal {
+ private final long fromId;
+ private final long toId;
+
+ public SparkPlanGraphEdgeInternal(long fromId, long toId) {
+ this.fromId = fromId;
+ this.toId = toId;
+ }
+
+ public long getFromId() {
+ return fromId;
+ }
+
+ public long getToId() {
+ return toId;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphInternal.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphInternal.java
new file mode 100644
index 0000000000..e4639ad106
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphInternal.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.graph;
+
+import java.util.List;
+
+/** Internal representation of a {@code org.apache.spark.sql.execution.ui.SparkPlanGraph}. */
+public class SparkPlanGraphInternal {
+ private final List<SparkPlanGraphNodeInternal> nodes;
+ private final List<SparkPlanGraphEdgeInternal> edges;
+ private final List<SparkPlanGraphNodeInternal> allNodes;
+
+ public SparkPlanGraphInternal(
+ List<SparkPlanGraphNodeInternal> nodes,
+ List<SparkPlanGraphEdgeInternal> edges,
+ List<SparkPlanGraphNodeInternal> allNodes) {
+ this.nodes = nodes;
+ this.edges = edges;
+ this.allNodes = allNodes;
+ }
+
+ public List<SparkPlanGraphNodeInternal> getNodes() {
+ return nodes;
+ }
+
+ public List<SparkPlanGraphEdgeInternal> getEdges() {
+ return edges;
+ }
+
+ public List<SparkPlanGraphNodeInternal> getAllNodes() {
+ return allNodes;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphNodeInternal.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphNodeInternal.java
new file mode 100644
index 0000000000..2486505280
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SparkPlanGraphNodeInternal.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.graph;
+
+import java.util.List;
+
+/** Internal representation of a {@code org.apache.spark.sql.execution.ui.SparkPlanGraphNode}. */
+public class SparkPlanGraphNodeInternal {
+ private final long id;
+ private final String name;
+ private final String desc;
+ private final List<SqlPlanMetricInternal> metrics;
+
+ public SparkPlanGraphNodeInternal(
+ long id, String name, String desc, List<SqlPlanMetricInternal> metrics) {
+ this.id = id;
+ this.name = name;
+ this.desc = desc;
+ this.metrics = metrics;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getAnonymizedName() {
+ if (name.contains("Scan")) {
+ String[] nameParts = name.split(" ", -1);
+ if (nameParts.length == 0) {
+ return "";
+ } else {
+ return nameParts[0];
+ }
+ } else {
+ return name;
+ }
+ }
+
+ public String getDesc() {
+ return desc;
+ }
+
+ public List<SqlPlanMetricInternal> getMetrics() {
+ return metrics;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("SparkPlanGraphNodeInternal{id=%d, name='%s', desc='%s'}", id, name, desc);
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SqlPlanMetricInternal.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SqlPlanMetricInternal.java
new file mode 100644
index 0000000000..372097c5ca
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/graph/SqlPlanMetricInternal.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.graph;
+
+/** Internal representation of a {@code org.apache.spark.sql.execution.ui.SqlPlanMetric}. */
+public class SqlPlanMetricInternal {
+ private final String name;
+ private final long accumulatorId;
+ private final String metricType;
+
+ public SqlPlanMetricInternal(String name, long accumulatorId, String metricType) {
+ this.name = name;
+ this.accumulatorId = accumulatorId;
+ this.metricType = metricType;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getAccumulatorId() {
+ return accumulatorId;
+ }
+
+ public String getMetricType() {
+ return metricType;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ChildSupportVisitor.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ChildSupportVisitor.java
new file mode 100644
index 0000000000..1b968c289c
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ChildSupportVisitor.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.gluten.qt.support.NotSupportedCategory.CHILD_NOT_SUPPORTED;
+
+/**
+ * A {@link GraphVisitor} that verifies child node support for each node in the {@link
+ * ExecutionDescription}. If any child of a node is {@link NotSupported}, this visitor marks that
+ * node as {@code NotSupported} too, indicating that the current node’s functionality depends on an
+ * unsupported child.
+ */
+public class ChildSupportVisitor extends GraphVisitor {
+ private static final String NOT_SUPPORTED_DESCRIPTION = "CHILD NOT SUPPORTED";
+
+ public ChildSupportVisitor(ExecutionDescription executionDescription) {
+ super(executionDescription);
+ }
+
+ @Override
+ protected void visitor(long nodeId) {
+ if (!resultSupportMap.containsKey(nodeId)) {
+ Map<Long, GlutenSupport> childrenSupportMap = new HashMap<>();
+ for (long childId : executionDescription.getChildren(nodeId)) {
+ childrenSupportMap.put(childId, resultSupportMap.get(childId));
+ }
+
+ ImmutableList<NotSupported> notSupportedChildrenId =
+ childrenSupportMap.entrySet().stream()
+ .filter(e -> e.getValue() instanceof NotSupported)
+ .map(e -> (NotSupported) e.getValue())
+ .collect(ImmutableList.toImmutableList());
+
+ GlutenSupport currentNodeSupport =
+ executionDescription.getNodeIdToGluttenSupportMap().get(nodeId);
+
+ final GlutenSupport resultSupport;
+ if (notSupportedChildrenId.isEmpty()) {
+ resultSupport = currentNodeSupport;
+ } else if (currentNodeSupport instanceof NotSupported) {
+ NotSupported ns = (NotSupported) currentNodeSupport;
+ ns.addReason(CHILD_NOT_SUPPORTED, NOT_SUPPORTED_DESCRIPTION);
+ resultSupport = ns;
+ } else {
+ resultSupport = new NotSupported(CHILD_NOT_SUPPORTED, NOT_SUPPORTED_DESCRIPTION);
+ }
+ resultSupportMap.put(nodeId, resultSupport);
+ }
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ClusterSupportVisitor.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ClusterSupportVisitor.java
new file mode 100644
index 0000000000..fb49c78448
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ClusterSupportVisitor.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.stream.Collectors;
+
+import static org.apache.gluten.qt.support.NotSupportedCategory.CLUSTER_NOT_SUPPORTED;
+
+/**
+ * {@link GraphVisitor} that evaluates support for entire clusters within the {@link
+ * ExecutionDescription}. If a node is part of a cluster:
+ *
+ * <ul>
+ * <li>If all nodes within the cluster are supported, this class marks each node in that cluster
+ * as supported.
+ * <li>If any node in the cluster is not supported, this class updates all nodes in that cluster
+ * to be {@link NotSupported} and records the relevant reasons.
+ * </ul>
+ *
+ * If a node does not belong to a cluster, its support status is taken directly from the {@code
+ * NodeIdToGluttenSupportMap}.
+ */
+public class ClusterSupportVisitor extends GraphVisitor {
+ public ClusterSupportVisitor(ExecutionDescription executionDescription) {
+ super(executionDescription);
+ }
+
+ @Override
+ protected void visitor(long nodeId) {
+ if (!resultSupportMap.containsKey(nodeId) && executionDescription.isNodeInCluster(nodeId)) {
+ ImmutableMap<Long, GlutenSupport> clusterSupport =
+ executionDescription.getCluster(nodeId).getNodes().stream()
+ .collect(
+ ImmutableMap.toImmutableMap(
+ n -> n.getId(),
+ n -> executionDescription.getNodeIdToGluttenSupportMap().get(n.getId())));
+ ImmutableMap<Long, NotSupported> notSupportedMap =
+ clusterSupport.entrySet().stream()
+ .filter(s -> s.getValue() instanceof NotSupported)
+ .collect(
+ ImmutableMap.toImmutableMap(e -> e.getKey(), e -> (NotSupported) e.getValue()));
+
+ if (notSupportedMap.isEmpty()) {
+ resultSupportMap.putAll(clusterSupport);
+ } else {
+ String newDescription =
+ notSupportedMap.keySet().stream()
+ .map(id -> executionDescription.getNode(id).getName())
+ .distinct()
+ .collect(Collectors.joining(":"));
+ ImmutableMap<Long, GlutenSupport> newEntries =
+ clusterSupport.entrySet().stream()
+ .collect(
+ ImmutableMap.toImmutableMap(
+ e -> e.getKey(),
+ e -> {
+ if (e.getValue() instanceof NotSupported) {
+ NotSupported ns = (NotSupported) e.getValue();
+ ns.addReason(CLUSTER_NOT_SUPPORTED, newDescription);
+ return ns;
+ } else {
+ return new NotSupported(CLUSTER_NOT_SUPPORTED, newDescription);
+ }
+ }));
+ resultSupportMap.putAll(newEntries);
+ }
+ } else if (!resultSupportMap.containsKey(nodeId)
+ && !executionDescription.isNodeInCluster(nodeId)) {
+ resultSupportMap.put(nodeId, executionDescription.getNodeIdToGluttenSupportMap().get(nodeId));
+ }
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/DurationMetric.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/DurationMetric.java
new file mode 100644
index 0000000000..076d3281a4
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/DurationMetric.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import java.time.Duration;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A utility class for parsing and storing durations from string representations like "123.45 s",
+ * "12 ms", or "1.5 h". If the input string cannot be parsed or is empty, the duration defaults to
+ * zero.
+ */
+public class DurationMetric {
+ private final Duration duration;
+ private static final Pattern PATTERN = Pattern.compile("([0-9.]+) (ms|h|m|s)");
+
+ public DurationMetric() {
+ this.duration = Duration.ZERO;
+ }
+
+ public DurationMetric(String string) {
+ this.duration = parseDuration(string);
+ }
+
+ private Duration parseDuration(String string) {
+ if (string.isEmpty()) {
+ return Duration.ZERO;
+ }
+
+ String[] splitString = string.split("\n", -1);
+ if (splitString.length > 2) {
+ return Duration.ZERO;
+ }
+
+ String valueString = splitString[splitString.length - 1];
+
+ Matcher matcher = PATTERN.matcher(valueString);
+
+ if (matcher.find() && matcher.groupCount() == 2) {
+ String value = matcher.group(1);
+ String unit = matcher.group(2);
+
+ try {
+ float parsedValue = Float.parseFloat(value);
+ switch (unit) {
+ case "h":
+ return Duration.ofMillis((long) (parsedValue * 60 * 60 * 1000));
+ case "m":
+ return Duration.ofMillis((long) (parsedValue * 60 * 1000));
+ case "s":
+ return Duration.ofMillis((long) (parsedValue * 1000));
+ case "ms":
+ return Duration.ofMillis((long) parsedValue);
+ default:
+ return Duration.ZERO;
+ }
+ } catch (NumberFormatException e) {
+ return Duration.ZERO;
+ }
+ } else {
+ return Duration.ZERO;
+ }
+ }
+
+ public Duration getDuration() {
+ return duration;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ExecutionDescription.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ExecutionDescription.java
new file mode 100644
index 0000000000..80731a34f5
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ExecutionDescription.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import org.apache.gluten.qt.graph.MetricInternal;
+import org.apache.gluten.qt.graph.SparkPlanGraphClusterInternal;
+import org.apache.gluten.qt.graph.SparkPlanGraphInternal;
+import org.apache.gluten.qt.graph.SparkPlanGraphNodeInternal;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.toCollection;
+
+/**
+ * A container describing the structure and relationships of a Spark plan graph, including:
+ *
+ * <ul>
+ * <li>A mapping of node IDs to their children nodes (edges)
+ * <li>A mapping of node IDs to the corresponding {@link SparkPlanGraphNodeInternal} instances
+ * <li>A mapping of accumulator IDs to their metric values (from {@link MetricInternal})
+ * <li>Cluster membership details, indicating which nodes belong to a {@link
+ * SparkPlanGraphClusterInternal}
+ * <li>The earliest non-cluster node ID, designated as the “start node”
+ * <li>A mapping of node IDs to their {@link GlutenSupport} statuses
+ * </ul>
+ *
+ * <p>The primary constructor processes a {@link SparkPlanGraphInternal} along with its metrics to
+ * build these mappings. Each node, cluster, and metric in the graph is recorded internally.
+ * Subclasses of {@link GraphVisitor} may then traverse this structure, updating or reading the node
+ * support statuses.
+ */
+public class ExecutionDescription {
+ private final Map<Long, List<Long>> nodeIdToEdges;
+ private final Map<Long, SparkPlanGraphNodeInternal> nodeIdToNodeMap;
+ private final Map<Long, String> accumulatorIdToMetricMap;
+ private final Map<Long, SparkPlanGraphClusterInternal> nodeIdToClusterMap;
+ private final long startNodeId;
+ private final Map<Long, GlutenSupport> nodeIdToGluttenSupportMap;
+
+ private ExecutionDescription(
+ Map<Long, List<Long>> nodeIdToEdges,
+ Map<Long, SparkPlanGraphNodeInternal> nodeIdToNodeMap,
+ Map<Long, String> accumulatorIdToMetricMap,
+ Map<Long, SparkPlanGraphClusterInternal> nodeIdToClusterMap,
+ long startNodeId,
+ Map<Long, GlutenSupport> nodeIdToGluttenSupportMap) {
+ this.nodeIdToEdges = nodeIdToEdges;
+ this.nodeIdToNodeMap = nodeIdToNodeMap;
+ this.accumulatorIdToMetricMap = accumulatorIdToMetricMap;
+ this.nodeIdToClusterMap = nodeIdToClusterMap;
+ this.startNodeId = startNodeId;
+ this.nodeIdToGluttenSupportMap = nodeIdToGluttenSupportMap;
+ }
+
+ ExecutionDescription withNewSupportMap(Map<Long, GlutenSupport> newSupportMap) {
+ return new ExecutionDescription(
+ nodeIdToEdges,
+ nodeIdToNodeMap,
+ accumulatorIdToMetricMap,
+ nodeIdToClusterMap,
+ startNodeId,
+ newSupportMap);
+ }
+
+ public ExecutionDescription(SparkPlanGraphInternal graph, List<MetricInternal> metrics) {
+ this.nodeIdToEdges =
+ graph.getEdges().stream()
+ .collect(
+ Collectors.groupingBy(
+ e -> e.getToId(),
+ Collectors.mapping(e -> e.getFromId(), toCollection(ArrayList::new))));
+ this.nodeIdToNodeMap =
+ graph.getAllNodes().stream()
+ .collect(ImmutableMap.toImmutableMap(SparkPlanGraphNodeInternal::getId, node -> node));
+
+ this.accumulatorIdToMetricMap =
+ metrics.stream()
+ .collect(
+ ImmutableMap.toImmutableMap(
+ MetricInternal::getKey, MetricInternal::getMetricValue));
+
+ this.nodeIdToClusterMap =
+ graph.getNodes().stream()
+ .filter(SparkPlanGraphClusterInternal.class::isInstance)
+ .map(SparkPlanGraphClusterInternal.class::cast)
+ .flatMap(
+ cluster ->
+ cluster.getNodes().stream().map(node -> Map.entry(node.getId(), cluster)))
+ .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+ this.startNodeId =
+ Collections.min(
+ nodeIdToNodeMap.entrySet().stream()
+ .filter(kv -> !(kv.getValue() instanceof SparkPlanGraphClusterInternal))
+ .map(Map.Entry::getKey)
+ .collect(ImmutableList.toImmutableList()));
+ this.nodeIdToGluttenSupportMap = new HashMap<>();
+ }
+
+ public long getStartNodeId() {
+ return startNodeId;
+ }
+
+ public Map<Long, GlutenSupport> getNodeIdToGluttenSupportMap() {
+ return nodeIdToGluttenSupportMap;
+ }
+
+ public List<Long> getChildren(long nodeId) {
+ return nodeIdToEdges.getOrDefault(nodeId, Collections.emptyList());
+ }
+
+ public SparkPlanGraphNodeInternal getNode(long nodeId) {
+ return nodeIdToNodeMap.get(nodeId);
+ }
+
+ public String getMetrics(long accumulatorId) {
+ return accumulatorIdToMetricMap.get(accumulatorId);
+ }
+
+ public boolean isNodeInCluster(long nodeId) {
+ return nodeIdToClusterMap.containsKey(nodeId);
+ }
+
+ public SparkPlanGraphClusterInternal getCluster(long nodeId) {
+ return nodeIdToClusterMap.get(nodeId);
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GlutenExecutionSupportReporter.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GlutenExecutionSupportReporter.java
new file mode 100644
index 0000000000..66e08636a6
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GlutenExecutionSupportReporter.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import java.time.Duration;
+import java.util.Map;
+
+/**
+ * Collects and reports information about the parts of a SQL workload that are supported by the
+ * Gluten engine.
+ *
+ * <p>Specifically, this class calculates the total SQL execution time, the portion of that time
+ * that is supported by Gluten, and identifies any operators requiring implementation for full
+ * support.
+ */
+public class GlutenExecutionSupportReporter {
+ private final Duration supportedSqlTime;
+ private final Duration totalSqlTime;
+ private final Map<String, ResultVisitor.UnsupportedImpact> operatorToImplement;
+
+ public GlutenExecutionSupportReporter(ExecutionDescription description) {
+ ExecutionDescription description1 = new NodeSupportVisitor(description).visitAndTag();
+ ExecutionDescription description2 = new ClusterSupportVisitor(description1).visitAndTag();
+ ExecutionDescription description3 = new ChildSupportVisitor(description2).visitAndTag();
+
+ ResultVisitor resultVisitor = new ResultVisitor(description3);
+ resultVisitor.visit();
+
+ this.supportedSqlTime = resultVisitor.getSupportedSqlTime();
+ this.totalSqlTime = resultVisitor.getTotalSqlTime();
+ this.operatorToImplement = resultVisitor.getUnsupportedOperatorImpactCostMap();
+ }
+
+ public Duration getSupportedSqlTime() {
+ return supportedSqlTime;
+ }
+
+ public Map<String, ResultVisitor.UnsupportedImpact> getOperatorToImplement() {
+ return operatorToImplement;
+ }
+
+ public Duration getTotalSqlTime() {
+ return totalSqlTime;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GlutenSupport.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GlutenSupport.java
new file mode 100644
index 0000000000..fe12ab2319
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GlutenSupport.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+/**
+ * Serves as a base class for indicating whether a specific node or operator is supported by Gluten.
+ * Subclasses, such as {@code Supported} or {@code NotSupported}, provide the specific status and
+ * any relevant details.
+ */
+public abstract class GlutenSupport {}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GraphVisitor.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GraphVisitor.java
new file mode 100644
index 0000000000..0dba1a20d7
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/GraphVisitor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An abstract class that defines a post-order traversal mechanism for a graph described by an
+ * {@link ExecutionDescription}. Subclasses implement the {@link #visitor(long)} method to specify
+ * how each node is processed.
+ *
+ * <p>During the traversal, this class keeps track of visited nodes to avoid redundant processing
+ * and accumulates a support map, which is used to augment the final {@link ExecutionDescription}
+ * instance.
+ */
+public abstract class GraphVisitor {
+ protected final ExecutionDescription executionDescription;
+ protected final Set<Long> visitedSet = new HashSet<>();
+ protected final Map<Long, GlutenSupport> resultSupportMap = new HashMap<>();
+
+ public GraphVisitor(ExecutionDescription executionDescription) {
+ this.executionDescription = executionDescription;
+ }
+
+ public ExecutionDescription visitAndTag() {
+ visit(executionDescription.getStartNodeId());
+ return executionDescription.withNewSupportMap(resultSupportMap);
+ }
+
+ // Visit in post order
+ protected void visit(long nodeId) {
+ if (!visitedSet.contains(nodeId)) {
+ for (long edge : executionDescription.getChildren(nodeId)) {
+ visit(edge);
+ }
+ visitor(nodeId);
+ visitedSet.add(nodeId);
+ }
+ }
+
+ protected abstract void visitor(long nodeId);
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NodeSupportVisitor.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NodeSupportVisitor.java
new file mode 100644
index 0000000000..bcb922558c
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NodeSupportVisitor.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import org.apache.gluten.qt.graph.SparkPlanGraphNodeInternal;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.apache.gluten.qt.support.NotSupportedCategory.NODE_NOT_SUPPORTED;
+
+/**
+ * {@link GraphVisitor} implementation that inspects each node in the {@code ExecutionDescription}
+ * to determine whether it is fully supported by Gluten. Specifically, this visitor classifies each
+ * node as either {@link Supported} or {@link NotSupported} based on:
+ *
+ * <ul>
+ * <li>The operator name (e.g., {@code BroadcastHashJoin}, {@code Scan})
+ * <li>Data formats (e.g., Parquet, ORC) and their unsupported types
+ * <li>Resource constraints (e.g., broadcast data size)
+ * </ul>
+ *
+ * The aggregated results are then stored in {@code resultSupportMap} and incorporated into a new
+ * {@link ExecutionDescription} instance.
+ */
+public class NodeSupportVisitor extends GraphVisitor {
+ private static final Pattern MEM_PATTERN =
+ Pattern.compile("^\\d+\\.?\\d*\\s*(EiB|PiB|TiB|GiB|MiB|KiB|B)\\b");
+ private static final Pattern SIZE_PATTERN = Pattern.compile("([\\d.]+)\\s*([a-zA-Z]+)");
+ private static final Pattern FORMAT_PATTERN = Pattern.compile("Format: ([^,]+)");
+ private static final Pattern SCHEMA_PATTERN = Pattern.compile("ReadSchema: struct<(.*)");
+ private static final ImmutableSet<String> FULLY_SUPPORTED_OPERATORS =
+ ImmutableSet.of(
+ "AdaptiveSparkPlan",
+ "TakeOrderedAndProject",
+ "Project",
+ "Filter",
+ "Window",
+ "Coalesce",
+ "CartesianProduct",
+ "HashAggregate",
+ "SortAggregate",
+ "Sort",
+ "ObjectHashAggregate",
+ "Expand",
+ "Union",
+ "ShuffleExchange",
+ "ShuffledHashJoin",
+ "SortMergeJoin",
+ "BroadcastExchange",
+ "GlobalLimit",
+ "LocalLimit",
+ "Generate",
+ "EvalPython",
+ "AQEShuffleRead",
+ "ReusedExchange",
+ "Subquery",
+ "SubqueryBroadcast",
+ "ShuffleQueryStage",
+ "Exchange",
+ "BroadcastQueryStage",
+ "ColumnarToRow",
+ "RowToColumnar",
+ "ReusedSubquery",
+ "ColumnarExchange",
+ "ColumnarBroadcastExchange",
+ "VeloxColumnarToRowExec",
+ "RowToVeloxColumnar",
+ "ColumnarSubqueryBroadcast",
+ "ColumnarUnion",
+ "InMemoryTableScan",
+ "BroadcastNestedLoopJoin",
+ "CollectLimit");
+
+ public NodeSupportVisitor(ExecutionDescription executionDescription) {
+ super(executionDescription);
+ }
+
+ @Override
+ protected void visitor(long nodeId) {
+ resultSupportMap.put(nodeId, isSupported(nodeId));
+ }
+
+ private String schemaConditions(String schema, List<String> conditions) {
+ return conditions.stream()
+ .map(cond -> ":" + cond)
+ .filter(schema::contains)
+ .collect(Collectors.joining(" and "))
+ .replace(":", "");
+ }
+
+ private Optional<String> extractSize(String input) {
+ Matcher matcher = MEM_PATTERN.matcher(input);
+
+ if (matcher.find()) {
+ return Optional.of(matcher.group());
+ } else {
+ return Optional.empty();
+ }
+ }
+
+ private Optional<BigInteger> stringToBytes(String sizeStr) {
+ Matcher matcher = SIZE_PATTERN.matcher(sizeStr);
+ if (matcher.matches()) {
+ BigDecimal size = new BigDecimal(matcher.group(1));
+ String unit = matcher.group(2);
+ switch (unit) {
+ case "EiB":
+ return Optional.of(size.multiply(BigDecimal.valueOf(1L << 60)).toBigInteger());
+ case "PiB":
+ return Optional.of(size.multiply(BigDecimal.valueOf(1L << 50)).toBigInteger());
+ case "TiB":
+ return Optional.of(size.multiply(BigDecimal.valueOf(1L << 40)).toBigInteger());
+ case "GiB":
+ return Optional.of(size.multiply(BigDecimal.valueOf(1L << 30)).toBigInteger());
+ case "MiB":
+ return Optional.of(size.multiply(BigDecimal.valueOf(1L << 20)).toBigInteger());
+ case "KiB":
+ return Optional.of(size.multiply(BigDecimal.valueOf(1L << 10)).toBigInteger());
+ case "B":
+ return Optional.of(size.toBigInteger());
+ default:
+ return Optional.empty();
+ }
+ }
+ return Optional.empty();
+ }
+
+ private static final ImmutableList<String> PARQUET_UNSUPPORTED_TYPES =
+ ImmutableList.of("struct", "map", "list", "tinyint");
+
+ private static final ImmutableList<String> ORC_UNSUPPORTED_TYPES =
+ ImmutableList.of("struct", "map", "list", "tinyint", "timestamp", "varchar", "char", "union");
+
+ private boolean checkTypeSupport(String dataType, List<String> unsupportedList) {
+ return unsupportedList.stream().map(s -> ":" + s).noneMatch(dataType::contains);
+ }
+
+ private GlutenSupport isSupported(long nodeId) {
+ SparkPlanGraphNodeInternal node = executionDescription.getNode(nodeId);
+ String name = node.getName();
+ String anonymizedName = node.getAnonymizedName();
+ String desc = node.getDesc();
+
+ if (FULLY_SUPPORTED_OPERATORS.contains(name)) {
+ return new Supported();
+ } else if (name.equals("BroadcastHashJoin")) {
+ boolean bhjSupported =
+ executionDescription.getChildren(nodeId).stream()
+ .map(executionDescription::getNode)
+ .filter(childNode -> childNode.getName().equals("BroadcastExchange"))
+ .flatMap(
+ childNode ->
+ childNode.getMetrics().stream()
+ .filter(metric -> metric.getName().contains("data size")))
+ .flatMap(
+ metric ->
+ Optional.ofNullable(
+ executionDescription.getMetrics(metric.getAccumulatorId()))
+ .stream())
+ .flatMap(
+ metricStr ->
+ Arrays.stream(metricStr.split("\n", -1))
+ .reduce((first, second) -> second)
+ .stream())
+ .flatMap(metricStr -> extractSize(metricStr).stream())
+ .flatMap(metricStr -> stringToBytes(metricStr).stream())
+ .anyMatch(size -> size.compareTo(BigInteger.valueOf(100L << 20)) <= 0);
+ return bhjSupported
+ ? new Supported()
+ : new NotSupported(NODE_NOT_SUPPORTED, "BHJ Not Supported");
+ } else if (name.contains("Scan")) {
+ Matcher formatMatcher = FORMAT_PATTERN.matcher(desc);
+ Matcher schemaMatcher = SCHEMA_PATTERN.matcher(desc);
+ String format = formatMatcher.find() ? formatMatcher.group(1) : "";
+ String schema = schemaMatcher.find() ? schemaMatcher.group(1) : "";
+ if (format.equals("Parquet") && checkTypeSupport(schema, PARQUET_UNSUPPORTED_TYPES)) {
+ return new Supported();
+ } else if (format.equals("ORC") && checkTypeSupport(schema, ORC_UNSUPPORTED_TYPES)) {
+ return new Supported();
+ } else if (format.equals("Parquet")) {
+ return new NotSupported(
+ NODE_NOT_SUPPORTED,
+ "Parquet with " + schemaConditions(schema, PARQUET_UNSUPPORTED_TYPES));
+ } else if (format.equals("ORC")) {
+ return new NotSupported(
+ NODE_NOT_SUPPORTED, "Orc with " + schemaConditions(schema, ORC_UNSUPPORTED_TYPES));
+ } else {
+ return new NotSupported(
+ NODE_NOT_SUPPORTED,
+ anonymizedName
+ + " with format "
+ + (format.isBlank() ? "UNKNOWN" : format)
+ + " not supported");
+ }
+ } else if (name.matches(".*Transformer$")) {
+ return new Supported();
+ } else {
+ return new NotSupported(NODE_NOT_SUPPORTED, anonymizedName + " not supported");
+ }
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NotSupported.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NotSupported.java
new file mode 100644
index 0000000000..300518f210
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NotSupported.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import org.jspecify.annotations.Nullable;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Indicates that a particular node or operator is not supported by Gluten, along with categorized
+ * reasons for the lack of support.
+ *
+ * <p>Each {@link NotSupportedCategory} in the internal map may have a corresponding reason
+ * (truncated to a maximum length for clarity). Additional reasons can be added via {@link
+ * #addReason(NotSupportedCategory, String)}.
+ */
+public class NotSupported extends GlutenSupport {
+ private final Map<NotSupportedCategory, String> notSupportedReasonMap = new HashMap<>();
+
+ public NotSupported(NotSupportedCategory category, String reason) {
+ notSupportedReasonMap.put(category, getFirstNChars(reason, 200));
+ }
+
+ public void addReason(NotSupportedCategory category, String reason) {
+ notSupportedReasonMap.put(category, getFirstNChars(reason, 200));
+ }
+
+ public boolean isNotSupported(NotSupportedCategory category) {
+ return notSupportedReasonMap.containsKey(category);
+ }
+
+ public String getCategoryReason(NotSupportedCategory category) {
+ return notSupportedReasonMap.get(category);
+ }
+
+ private static @Nullable String getFirstNChars(String input, int n) {
+ if (input == null) {
+ return null;
+ }
+ return input.length() > n ? input.substring(0, n) : input;
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NotSupportedCategory.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NotSupportedCategory.java
new file mode 100644
index 0000000000..a6acc0b0b6
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/NotSupportedCategory.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+/** Specifies categories of reasons why a node or operator might not be supported by Gluten. */
+public enum NotSupportedCategory {
+ NODE_NOT_SUPPORTED, // The node itself is unsupported.
+ CLUSTER_NOT_SUPPORTED, // A group or cluster of nodes is unsupported.
+ CHILD_NOT_SUPPORTED // One or more child nodes are unsupported, rendering the parent effectively
+ // unsupported.
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ResultVisitor.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ResultVisitor.java
new file mode 100644
index 0000000000..7ee1722ea1
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/ResultVisitor.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.gluten.qt.support.NotSupportedCategory.NODE_NOT_SUPPORTED;
+
+/**
+ * A {@link GraphVisitor} implementation that evaluates each node (and its cluster) in an {@link
+ * ExecutionDescription} to determine total versus supported SQL time.
+ *
+ * <p>It tracks any operators marked as {@code NotSupported} (specifically under the {@link
+ * NotSupportedCategory#NODE_NOT_SUPPORTED} category), aggregates their impact cost, and calculates
+ * cumulative durations for supported and unsupported portions of the SQL plan.
+ */
+public class ResultVisitor extends GraphVisitor {
+
+ public ResultVisitor(ExecutionDescription executionDescription) {
+ super(executionDescription);
+ }
+
+ public void visit() {
+ visit(executionDescription.getStartNodeId());
+ }
+
+ public ExecutionDescription visitAndTag() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Represents the cumulative impact of unsupported operators on CPU time and operator count.
+ *
+ * <p>This class aggregates the total CPU duration spent on unsupported operators and tracks how
+ * many such operators were encountered.
+ */
+ public static class UnsupportedImpact {
+ private Duration cumulativeCpuDuration = Duration.ZERO;
+ private long count = 0L;
+
+ public Duration getCumulativeCpuDuration() {
+ return cumulativeCpuDuration;
+ }
+
+ public long getCount() {
+ return count;
+ }
+
+ public void addCpuDuration(Duration duration) {
+ cumulativeCpuDuration = cumulativeCpuDuration.plus(duration);
+ }
+
+ public void increment() {
+ count++;
+ }
+
+ @CanIgnoreReturnValue
+ public UnsupportedImpact add(UnsupportedImpact other) {
+ this.cumulativeCpuDuration = this.cumulativeCpuDuration.plus(other.cumulativeCpuDuration);
+ this.count += other.count;
+ return this;
+ }
+ }
+
+ private final Set<Long> visitedCluster = new HashSet<>();
+ private Duration supportedSqlTime = Duration.ZERO;
+ private Duration totalSqlTime = Duration.ZERO;
+ private final Map<Long, List<Long>> nodeIdToUnsupportedOperatorsMap = new HashMap<>();
+ private final Map<Long, UnsupportedImpact> unsupportedOperatorImpactCostMap = new HashMap<>();
+ private final Map<Long, String> unsupportedOperatorReasonMap = new HashMap<>();
+
+ private boolean isOperatorNotSupported(long nodeId) {
+ GlutenSupport gs = executionDescription.getNodeIdToGluttenSupportMap().get(nodeId);
+ return gs instanceof NotSupported && ((NotSupported) gs).isNotSupported(NODE_NOT_SUPPORTED);
+ }
+
+ private NotSupported getNotSupportedOperator(long nodeId) {
+ GlutenSupport gs = executionDescription.getNodeIdToGluttenSupportMap().get(nodeId);
+ return (NotSupported) gs;
+ }
+
+ private DurationMetric getClusterDurationFromNodeId(long nodeId) {
+ return executionDescription.getCluster(nodeId).getMetrics().stream()
+ .flatMap(
+ metric ->
+ Optional.ofNullable(executionDescription.getMetrics(metric.getAccumulatorId()))
+ .map(DurationMetric::new).stream())
+ .findFirst()
+ .orElse(new DurationMetric());
+ }
+
+ @Override
+ protected void visitor(long nodeId) {
+ final DurationMetric durationMetric;
+
+ // Add towards duration only once per cluster
+ if (executionDescription.isNodeInCluster(nodeId)) {
+ long clusterId = executionDescription.getCluster(nodeId).getId();
+ durationMetric = getClusterDurationFromNodeId(nodeId);
+ if (!visitedCluster.contains(clusterId)) {
+ if (executionDescription.getNodeIdToGluttenSupportMap().get(nodeId) instanceof Supported) {
+ supportedSqlTime = supportedSqlTime.plus(durationMetric.getDuration());
+ }
+ totalSqlTime = totalSqlTime.plus(durationMetric.getDuration());
+ visitedCluster.add(clusterId);
+ }
+ } else {
+ durationMetric = new DurationMetric();
+ }
+
+ // This list will contain IDs of the current node and its children whose category is
+ // NODE_NOT_SUPPORTED
+ List<Long> currentNodeUnsupportedOperators = new ArrayList<>();
+ if (isOperatorNotSupported(nodeId)) {
+ currentNodeUnsupportedOperators.add(nodeId);
+ unsupportedOperatorImpactCostMap.put(nodeId, new UnsupportedImpact());
+ unsupportedOperatorReasonMap.put(
+ nodeId, getNotSupportedOperator(nodeId).getCategoryReason(NODE_NOT_SUPPORTED));
+ }
+
+ // Add all children whose category is NODE_NOT_SUPPORTED
+ for (long childId : executionDescription.getChildren(nodeId)) {
+ List<Long> childUnsupportedCauseList = nodeIdToUnsupportedOperatorsMap.get(childId);
+ currentNodeUnsupportedOperators.addAll(childUnsupportedCauseList);
+ }
+
+ // Penalize the current node and children nodes if they are NODE_NOT_SUPPORTED.
+ // The penalty is added because the children being NODE_NOT_SUPPORTED are preventing current
+ // node to be pushed to native.
+ currentNodeUnsupportedOperators.forEach(
+ id -> {
+ UnsupportedImpact currentCost = unsupportedOperatorImpactCostMap.get(id);
+ currentCost.addCpuDuration(durationMetric.getDuration());
+ currentCost.increment();
+ });
+
+ nodeIdToUnsupportedOperatorsMap.put(nodeId, currentNodeUnsupportedOperators);
+ }
+
+ public Duration getSupportedSqlTime() {
+ return supportedSqlTime;
+ }
+
+ public Duration getTotalSqlTime() {
+ return totalSqlTime;
+ }
+
+ public Map<String, UnsupportedImpact> getUnsupportedOperatorImpactCostMap() {
+ return unsupportedOperatorImpactCostMap.entrySet().stream()
+ .collect(
+ Collectors.groupingBy(
+ id -> unsupportedOperatorReasonMap.get(id.getKey()),
+ Collectors.collectingAndThen(
+ Collectors.reducing(
+ new UnsupportedImpact(), Map.Entry::getValue, UnsupportedImpact::add),
+ impact -> impact)));
+ }
+}
diff --git a/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/Supported.java b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/Supported.java
new file mode 100644
index 0000000000..f03aae44bc
--- /dev/null
+++ b/tools/qualification-tool/src/main/java/org/apache/gluten/qt/support/Supported.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support;
+
+/**
+ * Indicates that a particular node or operator is fully supported by Gluten.
+ *
+ * <p>This class simply extends {@link GlutenSupport} and overrides {@link #toString()} to identify
+ * the support status as “Supported.”
+ */
+public class Supported extends GlutenSupport {
+
+ @Override
+ public String toString() {
+ return ",,," + "Supported";
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/AppStatusListener.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/AppStatusListener.scala
new file mode 100644
index 0000000000..b6d60241e4
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/AppStatusListener.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+import org.apache.spark.SparkConf
+import org.apache.spark.util.kvstore.KVStore
+
+import ReflectionUtils.accessPrivateClass
+
+class AppStatusListener(conf: SparkConf, kvstore: KVStore) extends SparkListener {
+ private val elemTrackStore =
+ accessPrivateClass("org.apache.spark.status.ElementTrackingStore", kvstore, conf)
+ private val listener = accessPrivateClass(
+ "org.apache.spark.status.AppStatusListener",
+ elemTrackStore,
+ conf,
+ false,
+ None,
+ None)
+
+ override def getListener: Any = listener
+
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/AppStatusStore.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/AppStatusStore.scala
new file mode 100644
index 0000000000..cd0c7b1110
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/AppStatusStore.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+import org.apache.spark.status.api.v1
+import org.apache.spark.util.kvstore.InMemoryStore
+
+import ReflectionUtils.{accessPrivateClass, accessPrivateMethod}
+
+import java.util.regex.Pattern
+
+import scala.collection.JavaConverters._
+
+class AppStatusStore(kvStore: InMemoryStore) {
+ private val appStatusStore =
+ accessPrivateClass("org.apache.spark.status.AppStatusStore", kvStore, None, None)
+ private lazy val applicationInfo = accessPrivateMethod(appStatusStore, "applicationInfo")
+ private val HOST_URL_PATTERN = Pattern.compile("gdpic-srvls-batch-([a-z0-9\\-]+)-m.*")
+
+ def getApplicationId: String = accessPrivateMethod(applicationInfo, "id").asInstanceOf[String]
+ def getApplicationName: String = accessPrivateMethod(applicationInfo, "name").asInstanceOf[String]
+
+ def getJobIdToTaskDurationMap: Seq[(Int, Long)] = {
+ val jobSeq = accessPrivateMethod(appStatusStore, "jobsList", null).asInstanceOf[Seq[_]]
+ val jobIdToTaskDurationMap = jobSeq.map {
+ js =>
+ val jobId = accessPrivateMethod(js, "jobId").asInstanceOf[Int]
+ val taskDuration = accessPrivateMethod(js, "stageIds")
+ .asInstanceOf[collection.Seq[Int]]
+ .flatMap(
+ accessPrivateMethod(
+ appStatusStore,
+ "stageData",
+ _,
+ false,
+ List[v1.TaskStatus]().asJava,
+ false,
+ Array.empty[Double]).asInstanceOf[Seq[_]])
+ .map(accessPrivateMethod(_, "executorRunTime").asInstanceOf[Long])
+ .sum
+ jobId -> taskDuration
+ }
+ jobIdToTaskDurationMap
+ }
+
+ def getBatchUuid: String = {
+ val envInfo = accessPrivateMethod(appStatusStore, "environmentInfo", null)
+ .asInstanceOf[v1.ApplicationEnvironmentInfo]
+ envInfo.sparkProperties
+ .find(_._1.equalsIgnoreCase("spark.driver.host"))
+ .map(_._2)
+ .flatMap {
+ host =>
+ val hostMatcher = HOST_URL_PATTERN.matcher(host)
+ if (hostMatcher.find()) Some(hostMatcher.group(1)) else None
+ }
+ .getOrElse("UNKNOWN")
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/QualificationTool.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/QualificationTool.scala
new file mode 100644
index 0000000000..279cdc07d8
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/QualificationTool.scala
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+import org.apache.gluten.qt.execution.{EVENT_FILE_PROCESS_TASK, Executor, FILE_LISTING_TASK, PriorityTask, Progress, RunState, TaskContext}
+import org.apache.gluten.qt.file.HadoopFileSource
+import org.apache.gluten.qt.support.GlutenApplicationSupportReporter
+import org.apache.gluten.qt.writer.{OperatorImpactReportWriter, PrimaryReportWriter}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.execution.ui.SQLAppStatusStore
+import org.apache.spark.util.kvstore.InMemoryStore
+
+object QualificationTool {
+
+ def main(args: Array[String]): Unit = try {
+ val conf = QualificationToolConfiguration.parseCommandLineArgs(args)
+ val sparkConf = new SparkConf(false)
+ sparkConf.set("spark.appStateStore.asyncTracking.enable", "false")
+
+ val primaryReportWriter = writer.PrimaryReportWriter(conf)
+ val operatorImpactReportWriter = writer.OperatorImpactReportWriter(conf)
+ val fileSource = file.HadoopFileSource(conf)
+ val executor = execution.Executor(conf)
+
+ executor.submitTask(
+ PriorityTask(
+ TaskContext(100, "File Listing", FILE_LISTING_TASK),
+ () => {
+ Executor.updateState(RunState("LISTING FILES - COMPLETED", Progress()))
+ fileSource.getFileStatuses.foreach {
+ fileStatuses =>
+ Executor.incrementState()
+ executor.submitTask(
+ PriorityTask(
+ TaskContext(10, fileStatuses.head.getPath.toString, EVENT_FILE_PROCESS_TASK),
+ () => {
+ val replayBus = new ReplayListenerBus()
+ val kvStore = new InMemoryStore()
+ replayBus.addListener(new SQLAppStatusListener(sparkConf, kvStore))
+ replayBus.addListener(new AppStatusListener(sparkConf, kvStore))
+ fileStatuses
+ .map(_.getPath)
+ .foreach {
+ eventPath =>
+ replayBus.replay(fileSource.getSource(eventPath), eventPath.toString)
+ }
+ generateReport(conf, kvStore, primaryReportWriter, operatorImpactReportWriter)
+ }
+ ))
+ }
+ }
+ ))
+
+ executor.waitAndDisplayStatus()
+ primaryReportWriter.closeFile()
+ operatorImpactReportWriter.closeFile()
+ fileSource.close()
+ println(s"Report has been written to ${primaryReportWriter.getFileString}")
+ println(
+ s"Unsupported Operators have been written to ${operatorImpactReportWriter.getFileString}")
+ } catch {
+ case e: Throwable =>
+ e.printStackTrace(System.out)
+ throw e;
+ }
+
+ private def generateReport(
+ conf: QualificationToolConfiguration,
+ kvStore: InMemoryStore,
+ reportWriter: PrimaryReportWriter,
+ impactReportWriter: OperatorImpactReportWriter
+ ): Unit = {
+ val sqlStore = new SQLAppStatusStore(kvStore)
+ val appStore = new AppStatusStore(kvStore)
+ val applicationReporter = support.GlutenApplicationSupportReporter(conf, sqlStore, appStore)
+ val applicationReport = applicationReporter.getReport
+ reportWriter.addReport(applicationReport)
+ impactReportWriter.addReportBatch(applicationReporter.getOperatorsToImplement)
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/QualificationToolConfiguration.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/QualificationToolConfiguration.scala
new file mode 100644
index 0000000000..a74b6a378f
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/QualificationToolConfiguration.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+import org.apache.commons.cli.{CommandLine, DefaultParser, Options}
+
+import java.nio.file.{Files, Path, Paths}
+import java.time.Instant
+import java.time.temporal._
+
+/** Default Configurations */
+case class QualificationToolConfiguration(
+ inputPath: String,
+ verbose: Boolean = false,
+ gcsKeys: String = "",
+ outputPath: Path,
+ threads: Int,
+ dateFilter: Long,
+ project: String
+)
+
+object QualificationToolConfiguration {
+ def parseCommandLineArgs(args: Array[String]): QualificationToolConfiguration = {
+ val options = new Options()
+
+ options.addOption("f", "input", true, "File / Folder for qualification tool")
+ options.addOption("k", "gcsKey", true, "Provide the path to GCS keys")
+ options.addOption("o", "output", true, "Path to write output")
+ options.addOption("t", "threads", true, "Processing Threads")
+ options.addOption("v", "verbose", false, "non verbose output")
+ options.addOption("p", "project", true, "Project Id")
+ options.addOption("d", "dateFilter", true, "Do not analyze files created before this date")
+
+ val parser = new DefaultParser()
+ val cmd: CommandLine = parser.parse(options, args)
+
+ val inputPath = cmd.getOptionValue("input")
+ if (inputPath == null) {
+ throw new RuntimeException("Input Path can not be null")
+ }
+
+ val outputPath = Option(cmd.getOptionValue("output"))
+ .map {
+ p =>
+ val path = Paths.get(p)
+ Files.createDirectories(path)
+ path
+ }
+ .getOrElse {
+ Files.createTempDirectory(
+ java.util.UUID.randomUUID.toString.replaceAll(":+", "_").replaceAll("/", "_"))
+ }
+
+ val verbose = !cmd.hasOption("v")
+
+ val gcsKeys = cmd.getOptionValue("k")
+
+ val threads = Option(cmd.getOptionValue("threads")).map(_.toInt).getOrElse(4)
+ val dateFilter = Option(cmd.getOptionValue("dateFilter"))
+ .map(df => Instant.parse(df + "T00:00:00.00Z").toEpochMilli)
+ .getOrElse {
+ // Analyze only for last 90 days
+ Instant.now().minus(90, ChronoUnit.DAYS).toEpochMilli
+ }
+
+ val project = Option(cmd.getOptionValue("project")).getOrElse("")
+
+ QualificationToolConfiguration(
+ inputPath = inputPath,
+ verbose = verbose,
+ gcsKeys = gcsKeys,
+ outputPath = outputPath,
+ threads = threads,
+ dateFilter = dateFilter,
+ project)
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/ReflectionUtils.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/ReflectionUtils.scala
new file mode 100644
index 0000000000..b436969d1d
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/ReflectionUtils.scala
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+import scala.reflect.runtime.{universe => ru}
+import scala.reflect.runtime.currentMirror
+import scala.util.{Failure, Success, Try}
+
+object ReflectionUtils {
+
+ def accessPrivateClass(className: String, args: Any*): Any = {
+ val classSymbol = currentMirror.staticClass(className)
+ val classMirror = currentMirror.reflectClass(classSymbol)
+ val constructorSymbol = classSymbol.primaryConstructor.asMethod
+ val constructorMethod = classMirror.reflectConstructor(constructorSymbol)
+ constructorMethod(args: _*)
+ }
+
+ def accessPrivateMethod(instance: Any, methodName: String, args: Any*): Any = {
+ val instanceMirror = currentMirror.reflect(instance)
+ val methodSymbols =
+ instanceMirror.symbol.typeSignature.member(ru.TermName(methodName)).asTerm.alternatives
+
+ val exceptions = scala.collection.mutable.ArrayBuffer[Throwable]()
+ methodSymbols.iterator
+ .map {
+ methodSymbol =>
+ val method = instanceMirror.reflectMethod(methodSymbol.asMethod)
+ val result = Try(method(args: _*))
+
+ result match {
+ case Failure(e) =>
+ exceptions += e
+ None
+ case Success(value) => Some(value)
+ }
+ }
+ .collectFirst { case Some(value) => value }
+ .getOrElse {
+ val newException = new IllegalArgumentException(s"Method not found for $methodName")
+ exceptions.foreach(newException.addSuppressed)
+ throw newException
+ }
+ }
+
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/ReplayListenerBus.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/ReplayListenerBus.scala
new file mode 100644
index 0000000000..850dd43ff1
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/ReplayListenerBus.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+import org.apache.gluten.qt.execution.{Executor, Progress, RunState}
+
+import ReflectionUtils.{accessPrivateClass, accessPrivateMethod}
+
+import scala.io.BufferedSource
+
+class ReplayListenerBus {
+ private val replayBus = accessPrivateClass("org.apache.spark.scheduler.ReplayListenerBus")
+
+ def addListener(listener: SparkListener): Unit = {
+ accessPrivateMethod(replayBus, "addListener", listener.getListener)
+ }
+
+ def replay(source: BufferedSource, file: String): Boolean = {
+ try {
+ Executor.updateState(RunState("DOWNLOADING"))
+ val lines = source.getLines()
+ Executor.updateState(RunState("PARSING", Progress()))
+ val processedLines = lines.map {
+ line =>
+ Executor.incrementState()
+ line
+ }
+ val replayEventsFilter: String => Boolean = (_: String) => true
+ accessPrivateMethod(replayBus, "replay", processedLines, file, false, replayEventsFilter)
+ .asInstanceOf[Boolean]
+ } catch {
+ case e: Throwable => throw new Exception("Failed to process file", e)
+ } finally {
+ source.close()
+ }
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/SQLAppStatusListener.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/SQLAppStatusListener.scala
new file mode 100644
index 0000000000..1039416f27
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/SQLAppStatusListener.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+import org.apache.spark.SparkConf
+import org.apache.spark.util.kvstore.KVStore
+
+import ReflectionUtils.accessPrivateClass
+
+class SQLAppStatusListener(conf: SparkConf, kvstore: KVStore) extends SparkListener {
+ private val elemTrackStore =
+ accessPrivateClass("org.apache.spark.status.ElementTrackingStore", kvstore, conf)
+ private val listener = accessPrivateClass(
+ "org.apache.spark.sql.execution.ui.SQLAppStatusListener",
+ conf,
+ elemTrackStore,
+ false)
+
+ override def getListener: Any = listener
+
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/SparkListener.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/SparkListener.scala
new file mode 100644
index 0000000000..6e240fa89f
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/SparkListener.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt
+
+trait SparkListener {
+ def getListener: Any
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/execution/Executor.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/execution/Executor.scala
new file mode 100644
index 0000000000..003560c934
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/execution/Executor.scala
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.execution
+
+import org.apache.gluten.qt.QualificationToolConfiguration
+import org.apache.gluten.qt.writer.WriteUtils
+
+import Executor.hasUpdate
+
+import java.io.{FileWriter, PrintWriter}
+import java.util.concurrent.{ConcurrentHashMap, PriorityBlockingQueue, ThreadFactory, ThreadPoolExecutor, TimeUnit}
+import java.util.concurrent.atomic.AtomicLong
+
+import scala.collection.JavaConverters._
+
+/**
+ * Provides a thread-based execution mechanism using a pool of worker threads to process tasks with
+ * assigned priorities. The entry point is the {@code Executor} class, which manages a
+ * {@link ThreadPoolExecutor} along with counters for submitted, finished, and failed tasks. Tasks
+ * are submitted as {@link PriorityTask} instances, each containing a {@link TaskContext} that
+ * includes its priority and task type. <p> The {@code Executor} displays progress information in a
+ * loop until all threads become {@code IDLE} or the pool is shut down. It also handles error
+ * logging and can be run in verbose mode to provide real-time updates. <p> The key components are:
+ * <ul> <li>{@code Executor} - Creates and manages a thread pool to execute {@link PriorityTask}
+ * s.</li> <li>{@code PriorityTask} - Wraps a runnable task with a priority, allowing
+ * higher-priority tasks to be processed first.</li> <li>{@code TaskContext} - Contains metadata for
+ * a task (priority, name, and type).</li> <li>{@code RunState} - Represents the state of a thread,
+ * such as "STARTED" or "IDLE," and can also track progress.</li> <li>{@code Progress} - Holds the
+ * current and end values for task progress monitoring.</li> </ul>
+ */
+case class Executor(conf: QualificationToolConfiguration) extends ThreadFactory {
+ private val corePoolSize = conf.threads
+ private val maxPoolSize = conf.threads
+ private val keepAliveTime = 1L
+ private val queue = new PriorityBlockingQueue[Runnable]()
+
+ private val executor =
+ new ThreadPoolExecutor(corePoolSize, maxPoolSize, keepAliveTime, TimeUnit.SECONDS, queue, this)
+
+ private val submittedCounter = new AtomicLong(0L)
+ private val finishedCounter = new AtomicLong(0L)
+ private val failedCounter = new AtomicLong(0L)
+
+ private val errorPath = conf.outputPath.resolve(s"qt.stderr")
+ WriteUtils.deleteFileIfPresent(errorPath.toAbsolutePath.toString)
+ private lazy val errorWriter = new PrintWriter(
+ new FileWriter(errorPath.toAbsolutePath.toString, true))
+ private def addError(ctx: TaskContext, e: Throwable): Unit = errorWriter.synchronized {
+ errorWriter.println(ctx.name)
+ e.printStackTrace(errorWriter)
+ errorWriter.println()
+ errorWriter.flush()
+ }
+
+ def submitTask(task: PriorityTask): Unit = {
+ if (task.ctx.taskType == EVENT_FILE_PROCESS_TASK) {
+ submittedCounter.incrementAndGet()
+ }
+ val taskWithStatus = task.copy(task = () => {
+ Executor.updateState(RunState("STARTED"))
+ try {
+ task.task()
+ if (task.ctx.taskType == EVENT_FILE_PROCESS_TASK) {
+ finishedCounter.incrementAndGet()
+ }
+ } catch {
+ case e: Throwable =>
+ addError(task.ctx, e)
+ if (task.ctx.taskType == EVENT_FILE_PROCESS_TASK) {
+ failedCounter.incrementAndGet()
+ }
+ }
+ Executor.updateState(RunState("IDLE"))
+ })
+ executor.execute(taskWithStatus)
+ }
+
+ override def newThread(r: Runnable): Thread = {
+ val thread = new Thread(r)
+ Executor.addThread(thread)
+ thread
+ }
+
+ def waitAndDisplayStatus(): Unit = {
+ var states = Executor.getStates
+
+ def moveCursorUp(n: Int): Unit = {
+ printf(s"\u001b[${n}A")
+ }
+
+ def clearLine(): Unit = {
+ printf("\u001b[2K")
+ }
+
+ def displayState(): Unit = {
+ val processing = submittedCounter.get() - (finishedCounter.get() + failedCounter.get())
+ clearLine()
+ println(
+ s"Processing $processing Processed $finishedCounter" +
+ (if (failedCounter.get() == 0) "" else s" Failed or Skipped $failedCounter")
+ )
+ states.zipWithIndex.foreach {
+ case (s, i) =>
+ clearLine()
+ println(s"T$i: $s")
+ }
+ }
+
+ def displayStateAndClean(): Unit = {
+ displayState()
+ moveCursorUp(states.size + 1)
+ }
+
+ while (!states.forall(_ == RunState("IDLE"))) {
+ if (conf.verbose && Executor.hasUpdate) {
+ displayStateAndClean()
+ hasUpdate = false
+ }
+ Thread.sleep(100)
+ states = Executor.getStates
+ }
+ displayState()
+ executor.shutdown()
+ errorWriter.close()
+ }
+}
+
+object Executor {
+ private val _stateStore = new ConcurrentHashMap[Thread, RunState]()
+ @volatile private var hasUpdate: Boolean = false
+ private def stateStore: ConcurrentHashMap[Thread, RunState] = {
+ hasUpdate = true
+ _stateStore
+ }
+
+ def updateState(runState: RunState): Unit = {
+ stateStore.compute(Thread.currentThread(), (_, _) => runState)
+ }
+
+ def incrementState(): Unit = {
+ stateStore.compute(
+ Thread.currentThread(),
+ (_, s) => {
+ s.progress.foreach(_.increment())
+ s
+ })
+ }
+
+ private def addThread(thread: Thread): Unit = {
+ stateStore.put(thread, RunState("NEW"))
+ }
+
+ private def getStates: Seq[RunState] = _stateStore.values().asScala.toSeq
+}
+
+case class PriorityTask(ctx: TaskContext, task: () => Unit)
+ extends Runnable
+ with Comparable[PriorityTask] {
+ override def run(): Unit = task()
+
+ override def compareTo(other: PriorityTask): Int = {
+ Integer.compare(
+ other.ctx.priority,
+ this.ctx.priority
+ ) // Higher priority tasks are executed first
+ }
+}
+
+sealed trait TaskType
+
+object EVENT_FILE_PROCESS_TASK extends TaskType
+object FILE_LISTING_TASK extends TaskType
+
+case class TaskContext(priority: Int, name: String, taskType: TaskType)
+
+case class RunState(state: String, progress: Option[Progress] = None) {
+ override def toString: String = s"$state ${progress.map(_.toString).getOrElse("")}"
+}
+
+object RunState {
+ def apply(state: String, end: Long): RunState = {
+ RunState(state, Some(Progress(end)))
+ }
+
+ def apply(state: String, progress: Progress): RunState = {
+ RunState(state, Some(progress))
+ }
+}
+
+case class Progress(end: Option[Long]) {
+ private var current = 0L
+
+ def increment(): Unit = {
+ current = current + 1
+ }
+
+ def update(newValue: Long): Unit = {
+ current = newValue
+ }
+
+ override def toString: String = s"$current${end.map("/" + _).getOrElse("")}"
+}
+
+object Progress {
+ def apply(end: Long): Progress = {
+ Progress(Some(end))
+ }
+
+ def apply(): Progress = {
+ Progress(None)
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/GcsFileSource.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/GcsFileSource.scala
new file mode 100644
index 0000000000..3ff805de9b
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/GcsFileSource.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.file
+
+import org.apache.gluten.qt.QualificationToolConfiguration
+
+import org.apache.hadoop.conf.Configuration
+
+/**
+ * A concrete implementation of {@link HadoopFileSource} for Google Cloud Storage (GCS). <p> This
+ * class customizes the Hadoop {@link Configuration} to support interaction with GCS using the
+ * Google Cloud Hadoop connector. It sets the necessary properties for accessing GCS, including the
+ * file system implementation and optional service account credentials. <p> Key features: <ul>
+ * <li>Configures the Hadoop environment to use {@code GoogleHadoopFileSystem} for GCS paths.</li>
+ * <li>Optionally sets up service account authentication using a JSON key file if {@code gcsKeys} is
+ * provided in the configuration.</li> </ul>
+ */
+case class GcsFileSource(conf: QualificationToolConfiguration) extends HadoopFileSource(conf) {
+ override protected def hadoopConf: Configuration = {
+ val hadoopConf = new Configuration()
+ val gcsKeys = conf.gcsKeys
+ hadoopConf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem")
+ hadoopConf.set("fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS")
+ if (gcsKeys != null && gcsKeys.nonEmpty) {
+ hadoopConf.set("fs.gs.auth.type", "SERVICE_ACCOUNT_JSON_KEYFILE")
+ hadoopConf.set("fs.gs.auth.service.account.json.keyfile", s"$gcsKeys")
+ }
+ hadoopConf
+ }
+
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/HadoopFileSource.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/HadoopFileSource.scala
new file mode 100644
index 0000000000..384fbde470
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/HadoopFileSource.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.file
+
+import org.apache.gluten.qt.{file, QualificationToolConfiguration}
+
+import org.apache.spark.deploy.history.{EventLogFileReader, EventLogFileWriter, RollingEventLogFilesWriter}
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
+
+import scala.io.{BufferedSource, Codec, Source}
+
+/**
+ * An abstract file source that leverages Hadoop’s {@link FileSystem} to traverse and read event log
+ * files. It uses the provided {@link QualificationToolConfiguration} to determine the input paths,
+ * date filters, and other settings. <p> Major functionalities include: <ul> <li>Resolving a list of
+ * valid event log file statuses by applying date filters, excluding temporary or hidden files, and
+ * handling rolling event log directories.</li> <li>Opening and returning a
+ * {@link scala.io.BufferedSource} for an event log file.</li> <li>Cleanly closing the underlying
+ * {@link FileSystem} resource.</li> </ul>
+ *
+ * @param conf
+ * the configuration specifying input paths, date filters, and other settings
+ */
+
+abstract class HadoopFileSource(conf: QualificationToolConfiguration) {
+ protected def inputPathStrs: Array[String] = conf.inputPath.split(",").filter(_.nonEmpty)
+ protected def hadoopConf = new Configuration()
+ private val fs: FileSystem = FileSystem.get(new Path(inputPathStrs(0)).toUri, hadoopConf)
+
+ def getFileStatuses: Iterator[Seq[FileStatus]] = {
+ inputPathStrs.distinct.toIterator.flatMap(ip => getFileStatuses(fs.getFileStatus(new Path(ip))))
+ }
+
+ private def getFileStatuses(status: FileStatus): Iterator[Seq[FileStatus]] = {
+ fs.listStatus(status.getPath).toIterator.flatMap {
+ s =>
+ if (
+ !s.isDirectory && !s.getPath.getName.startsWith(
+ ".") && status.getModificationTime >= conf.dateFilter && !s.getPath.getName.endsWith(
+ EventLogFileWriter.IN_PROGRESS)
+ ) {
+ Iterator(Seq(s))
+ } else if (
+ RollingEventLogFilesWriter.isEventLogDir(
+ s) && status.getModificationTime >= conf.dateFilter
+ ) {
+ Iterator(EventLogFileReader.apply(fs, s).get.listEventLogFiles)
+ } else if (s.isDirectory) {
+ fs.listStatus(s.getPath).toIterator.flatMap(getFileStatuses)
+ } else
+ {
+ Iterator[Seq[FileStatus]]()
+ }.filter(_.isEmpty)
+ }
+ }
+
+ def getSource(path: Path): BufferedSource = {
+ val inputStream = EventLogFileReader.openEventLog(path, fs)
+ Source.fromInputStream(inputStream)(Codec.UTF8)
+ }
+
+ def close(): Unit = fs.close()
+}
+
+object HadoopFileSource {
+ def apply(conf: QualificationToolConfiguration): HadoopFileSource = {
+ conf.inputPath match {
+ case f if f.startsWith("gs://") => GcsFileSource(conf)
+ case f if f.startsWith("file://") => file.LocalFsFileSource(conf)
+ case f if f.startsWith("/") => file.LocalFsFileSource(conf)
+ }
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/LocalFsFileSource.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/LocalFsFileSource.scala
new file mode 100644
index 0000000000..92846267a0
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/file/LocalFsFileSource.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.file
+
+import org.apache.gluten.qt.QualificationToolConfiguration
+
+/**
+ * A concrete implementation of {@link HadoopFileSource} for local file systems. <p> It extends the
+ * base functionality of {@code HadoopFileSource} by ensuring that all input paths are correctly
+ * prefixed with {@code file://} if they start with a forward slash, indicating a local file system
+ * path.
+ */
+
+case class LocalFsFileSource(conf: QualificationToolConfiguration) extends HadoopFileSource(conf) {
+ override protected def inputPathStrs: Array[String] = {
+ super.inputPathStrs.map(
+ ip =>
+ if (ip.startsWith("/")) {
+ s"file://${conf.inputPath}"
+ } else {
+ ip
+ })
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/support/GlutenApplicationSupportReporter.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/support/GlutenApplicationSupportReporter.scala
new file mode 100644
index 0000000000..22d2371f9d
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/support/GlutenApplicationSupportReporter.scala
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support
+
+import org.apache.gluten.qt.{writer, AppStatusStore, QualificationToolConfiguration}
+import org.apache.gluten.qt.execution.{Executor, RunState}
+import org.apache.gluten.qt.graph.{MetricInternal, SparkPlanGraphClusterInternal, SparkPlanGraphEdgeInternal, SparkPlanGraphInternal, SparkPlanGraphNodeInternal, SqlPlanMetricInternal}
+import org.apache.gluten.qt.writer.{ApplicationReport, ImpactReport}
+
+import org.apache.spark.sql.execution.ui.{SparkPlanGraph, SparkPlanGraphCluster, SparkPlanGraphNode, SQLAppStatusStore}
+
+import ResultVisitor.UnsupportedImpact
+
+import scala.collection.JavaConverters._
+
+/**
+ * Generates a detailed support report for a Spark application by analyzing its SQL execution plans.
+ * This includes identifying unsupported operators, calculating supported and unsupported SQL time,
+ * and providing an overall application report. <p> Major functionalities include: <ul>
+ * <li>Retrieving and analyzing execution plans for SQL queries via {@link SQLAppStatusStore} .</li>
+ * <li>Computing task durations for SQL and non-SQL operations.</li> <li>Determining supported and
+ * unsupported SQL time using {@link GlutenExecutionSupportReporter} .</li> <li>Aggregating operator
+ * impact reports for unsupported features.</li> <li>Compiling a summary {@link ApplicationReport} ,
+ * including task time breakdown and SQL support percentages.</li> </ul> <p>
+ *
+ * @param conf
+ * the configuration object providing project and path details
+ * @param sqlStore
+ * the SQL status store for accessing execution details
+ * @param appStore
+ * the application status store for task and application metadata
+ */
+case class GlutenApplicationSupportReporter(
+ conf: QualificationToolConfiguration,
+ sqlStore: SQLAppStatusStore,
+ appStore: AppStatusStore) {
+ private lazy val sqlJobIds = sqlStore
+ .executionsList()
+ .map(_.executionId)
+ .flatMap(sqlStore.execution)
+ .flatMap(_.jobs)
+ .map(_._1)
+ .toSet
+ private lazy val (sqlJobToDurationMap, jobToDurationMap) =
+ appStore.getJobIdToTaskDurationMap.partition(x => sqlJobIds.contains(x._1))
+ private lazy val sqlTaskTime = sqlJobToDurationMap.map(_._2).sum
+ private lazy val otherTaskTime = jobToDurationMap.map(_._2).sum
+ private lazy val totalTaskTime = sqlTaskTime + otherTaskTime
+ private lazy val applicationId = appStore.getApplicationId
+ private lazy val applicationName = appStore.getApplicationName
+ private lazy val batchUuid = appStore.getBatchUuid
+
+ private lazy val executionIds = sqlStore.executionsList().map(_.executionId)
+ Executor.updateState(RunState("PROCESSING", executionIds.size))
+ private lazy val executionResult = executionIds
+ .map {
+ executionId =>
+ Executor.incrementState()
+ val graph = sqlStore.planGraph(executionId)
+ val metrics = sqlStore.executionMetrics(executionId)
+ val graphDescription = new ExecutionDescription(
+ GlutenApplicationSupportReporter.toSparkPlanGraphInternal(graph),
+ GlutenApplicationSupportReporter.toMetricsInternal(metrics)
+ )
+ val executionSupport = new GlutenExecutionSupportReporter(graphDescription)
+ SupportReport(
+ executionId,
+ executionSupport.getSupportedSqlTime,
+ executionSupport.getTotalSqlTime,
+ executionSupport.getOperatorToImplement.asScala.toMap
+ )
+ }
+ private lazy val operatorToImplement = executionResult
+ .flatMap(e => e.operatorsToImplement)
+ .groupBy(_._1)
+ .mapValues(_.map(_._2).reduceOption(_.add(_)).getOrElse(new UnsupportedImpact()))
+ .toSeq
+ .map(r => writer.ImpactReport(r._1, r._2))
+ private lazy val supportedSqlTime = executionResult.map(e => e.supportedSqlTime.toMillis).sum
+ private lazy val totalSqlTime = executionResult.map(e => e.totalSqlTime.toMillis).sum
+
+ def getOperatorsToImplement: Seq[ImpactReport] = operatorToImplement
+
+ def getReport: ApplicationReport = {
+ val rddPercentage = if (totalTaskTime == 0) 0.0 else otherTaskTime.toDouble / totalTaskTime
+ val unsupportedSqlPercentage =
+ if (totalSqlTime == 0) 0.0
+ else ((totalSqlTime - supportedSqlTime).toDouble / totalSqlTime) * (1 - rddPercentage)
+ val supportedSqlPercentage =
+ if (totalSqlTime == 0) 0.0
+ else (supportedSqlTime.toDouble / totalSqlTime) * (1 - rddPercentage)
+ ApplicationReport(
+ applicationId,
+ applicationName,
+ batchUuid,
+ totalTaskTime,
+ rddPercentage,
+ unsupportedSqlPercentage,
+ supportedSqlPercentage
+ )
+ }
+}
+
+object GlutenApplicationSupportReporter {
+ def toSparkPlanGraphInternal(graph: SparkPlanGraph): SparkPlanGraphInternal = {
+ new SparkPlanGraphInternal(
+ toSparkPlanGraphNodesInternal(graph.nodes),
+ graph.edges.map(e => new SparkPlanGraphEdgeInternal(e.fromId, e.toId)).asJava,
+ toSparkPlanGraphNodesInternal(graph.allNodes)
+ )
+ }
+
+ def toSparkPlanGraphNodesInternal(
+ nodes: Seq[SparkPlanGraphNode]): java.util.List[SparkPlanGraphNodeInternal] = {
+ nodes.map {
+ node =>
+ val metric = node.metrics.map(toSqlPlanMetricInternal).asJava
+ node match {
+ case cluster: SparkPlanGraphCluster =>
+ new SparkPlanGraphClusterInternal(
+ cluster.id,
+ cluster.name,
+ cluster.desc,
+ toSparkPlanGraphNodesInternal(cluster.nodes),
+ metric
+ )
+ case node =>
+ new SparkPlanGraphNodeInternal(node.id, node.name, node.desc, metric)
+ }
+ }.asJava
+ }
+
+ def toSqlPlanMetricInternal(
+ metric: org.apache.spark.sql.execution.ui.SQLPlanMetric): SqlPlanMetricInternal = {
+ new SqlPlanMetricInternal(metric.name, metric.accumulatorId, metric.metricType)
+ }
+
+ def toMetricsInternal(metrics: Map[Long, String]): java.util.List[MetricInternal] = {
+ metrics.map(m => new MetricInternal(m._1, m._2)).toList.asJava
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/support/SupportReport.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/support/SupportReport.scala
new file mode 100644
index 0000000000..6a4261eb5f
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/support/SupportReport.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.support
+
+import java.time.Duration
+
+/** Represents a support analysis report for a single SQL execution. */
+
+case class SupportReport(
+ executionId: Long,
+ supportedSqlTime: Duration,
+ totalSqlTime: Duration,
+ operatorsToImplement: Map[String, ResultVisitor.UnsupportedImpact])
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ApplicationReport.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ApplicationReport.scala
new file mode 100644
index 0000000000..6ec939dc81
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ApplicationReport.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.writer
+
+case class ApplicationReport(
+ applicationId: String,
+ applicationName: String,
+ batchUuidd: String,
+ totalTaskTime: Long,
+ rddPercentage: Double,
+ unsupportedSqlPercentage: Double,
+ supportedSqlPercentage: Double
+) extends Report {
+ private def getSupportedTaskTime: Long = (totalTaskTime * supportedSqlPercentage).toLong
+
+ private val SPEED_BOOST = 0.3
+
+ override def toTSVLine: String = Seq(
+ s"$applicationId",
+ s"$applicationName",
+ s"$batchUuidd",
+ s"${round(rddPercentage * 100)}%",
+ s"${round(unsupportedSqlPercentage * 100)}%",
+ s"$totalTaskTime",
+ s"$getSupportedTaskTime",
+ s"${round(supportedSqlPercentage * 100)}%",
+ s"${supportedSqlPercentage >= 0.7}",
+ s"${round(supportedSqlPercentage * SPEED_BOOST * 100)}%"
+ ).filter(_.nonEmpty).map(_.replaceAll("\t", " ")).mkString("\t")
+
+ private def round(percentage: Double, points: Int = 1): BigDecimal =
+ BigDecimal(Math.floor(percentage * Math.pow(10, points)) / Math.pow(10, points))
+ .setScale(points, BigDecimal.RoundingMode.DOWN)
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ImpactReport.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ImpactReport.scala
new file mode 100644
index 0000000000..41c1b0977e
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ImpactReport.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.writer
+
+import org.apache.gluten.qt.support.ResultVisitor.UnsupportedImpact
+
+case class ImpactReport(operatorName: String, impact: UnsupportedImpact) extends Report {
+ private val cumulativeCpuMs = impact.getCumulativeCpuDuration.toMillis
+ private val count = impact.getCount
+
+ override def toTSVLine: String =
+ Seq(s"$operatorName", s"$cumulativeCpuMs", s"$count").mkString("\t")
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/OperatorImpactReportWriter.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/OperatorImpactReportWriter.scala
new file mode 100644
index 0000000000..b5675dbe97
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/OperatorImpactReportWriter.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.writer
+
+import org.apache.gluten.qt.QualificationToolConfiguration
+
+import scala.collection.mutable
+
+case class OperatorImpactReportWriter(conf: QualificationToolConfiguration)
+ extends ReportWriter[ImpactReport](conf = conf, name = "UnsupportedOperators") {
+ override def getHeader: String = Seq(
+ "unsupportedOperator", // 0
+ "cumulativeCpuMs", // 1
+ "count" // 2
+ ).mkString("\t")
+
+ override def doPostProcess(lines: Iterator[String]): Iterator[String] = {
+ val sumMap = mutable.Map[String, (Long, Long)]()
+ for (line <- lines) {
+ val Array(col1, col2, col3) = line.split("\t").map(_.trim)
+ val cpu = col2.toLong
+ val count = col3.toLong
+ val current = sumMap.getOrElse(col1, (0L, 0L))
+ sumMap(col1) = (current._1 + cpu, current._2 + count)
+ }
+ sumMap.toSeq
+ .sortBy(_._2)(Ordering.Tuple2(Ordering.Long.reverse, Ordering.Long.reverse))
+ .map(kv => s"${kv._1}\t${kv._2._1}\t${kv._2._2}")
+ .toIterator
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/PrimaryReportWriter.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/PrimaryReportWriter.scala
new file mode 100644
index 0000000000..7f844a6dff
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/PrimaryReportWriter.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.writer
+
+import org.apache.gluten.qt.QualificationToolConfiguration
+
+case class PrimaryReportWriter(conf: QualificationToolConfiguration)
+ extends ReportWriter[ApplicationReport](conf, "AppsRecommendedForBoost") {
+ override def getHeader: String = Seq(
+ "applicationId", // 0
+ "applicationName", // 1
+ "batchUuid", // 2
+ "rddPercentage", // 3
+ "unsupportedSqlPercentage", // 4
+ "totalTaskTime", // 5
+ "supportedTaskTime", // 6
+ "supportedSqlPercentage", // 7
+ "recommendedForBoost", // 8
+ "expectedRuntimeReduction" // 9
+ ).filter(_.nonEmpty).mkString("\t")
+
+ override def doPostProcess(lines: Iterator[String]): Iterator[String] = {
+ lines
+ .map(_.split("\t").map(_.trim))
+ .toSeq
+ .sortBy(a => (a(8).toBoolean, a(6).toLong))(
+ Ordering.Tuple2(Ordering.Boolean.reverse, Ordering.Long.reverse))
+ .map(_.mkString("\t"))
+ .toIterator
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/Report.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/Report.scala
new file mode 100644
index 0000000000..c5ae98f60d
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/Report.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.writer
+
+trait Report {
+ def toTSVLine: String
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ReportWriter.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ReportWriter.scala
new file mode 100644
index 0000000000..4a32f6d852
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/ReportWriter.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.writer
+
+import org.apache.gluten.qt.QualificationToolConfiguration
+
+import java.io.{BufferedWriter, FileWriter}
+
+import scala.io.Source
+
+abstract class ReportWriter[T <: Report](conf: QualificationToolConfiguration, name: String) {
+ private val tempPath = conf.outputPath.resolve(s"${name}_t.tsv")
+ private val path = WriteUtils.getUniquePath(conf.outputPath.resolve(s"$name.tsv"))
+ WriteUtils.deleteFileIfPresent(tempPath.toAbsolutePath.toString)
+ WriteUtils.deleteFileIfPresent(path.toAbsolutePath.toString)
+ private val writer: BufferedWriter = new BufferedWriter(
+ new FileWriter(tempPath.toAbsolutePath.toString, true))
+ addLine(getHeader)
+
+ def getHeader: String
+
+ def addReport(report: T): Unit = {
+ addLine(report.toTSVLine)
+ }
+
+ def addReportBatch(reportBatch: Seq[T]): Unit = {
+ reportBatch.foreach(addReport)
+ }
+
+ private def addLine(line: String): Unit = writer.synchronized {
+ writer.write(line)
+ writer.newLine()
+ writer.flush()
+ }
+
+ def closeFile(): Unit = {
+ if (writer != null) {
+ writer.flush()
+ writer.close()
+ }
+ postProcess()
+ }
+
+ def getFileString: String = path.toAbsolutePath.toString
+
+ def doPostProcess(lines: Iterator[String]): Iterator[String]
+
+ private def postProcess(): Unit = {
+ val filename = tempPath.toAbsolutePath.toString
+ val source = Source.fromFile(filename)
+ try {
+ val lines = source.getLines().drop(1)
+ val newLines = doPostProcess(lines)
+ val writer = new BufferedWriter(new FileWriter(path.toAbsolutePath.toString, true))
+ writer.write(getHeader)
+ writer.newLine()
+ while (newLines.hasNext) {
+ writer.write(newLines.next())
+ writer.newLine()
+ }
+ writer.flush()
+ writer.close()
+ } finally {
+ source.close()
+ }
+ WriteUtils.deleteFileIfPresent(filename)
+ }
+}
diff --git a/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/WriteUtils.scala b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/WriteUtils.scala
new file mode 100644
index 0000000000..cc1e115b5a
--- /dev/null
+++ b/tools/qualification-tool/src/main/scala/org/apache/gluten/qt/writer/WriteUtils.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt.writer
+
+import java.io.File
+import java.nio.file.Path
+
+object WriteUtils {
+ def deleteFileIfPresent(filePath: String): Boolean = {
+ val file = new File(filePath)
+ if (file.exists()) {
+ file.delete()
+ } else {
+ false
+ }
+ }
+
+ def getUniquePath(filePath: Path): Path = {
+ if (new File(filePath.toUri.getPath).exists()) {
+ var counter = 1
+ val splitPath = filePath.getFileName.toString.split("[.]")
+ val namePart = splitPath.init.mkString(".")
+ val extensionPart = splitPath.last
+ var newFilePath = filePath.getParent.resolve(s"${namePart}_$counter.$extensionPart")
+ while (new File(newFilePath.toUri.getPath).exists()) {
+ counter = counter + 1
+ newFilePath = filePath.getParent.resolve(s"${namePart}_$counter.$extensionPart")
+ }
+ newFilePath
+ } else {
+ filePath
+ }
+ }
+}
diff --git a/tools/qualification-tool/src/test/java/org/apache/gluten/qt/QualificationToolTest.java b/tools/qualification-tool/src/test/java/org/apache/gluten/qt/QualificationToolTest.java
new file mode 100644
index 0000000000..c998ccade3
--- /dev/null
+++ b/tools/qualification-tool/src/test/java/org/apache/gluten/qt/QualificationToolTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.qt;
+
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+public class QualificationToolTest {
+ private boolean areFileContentsSame(Path path1, Path path2) throws IOException {
+ byte[] file1Bytes = Files.readAllBytes(path1);
+ byte[] file2Bytes = Files.readAllBytes(path2);
+
+ return Arrays.equals(file1Bytes, file2Bytes);
+ }
+
+ @Test
+ public void testQualificationToolMain() throws IOException, URISyntaxException {
+ String eventFilePath =
+ new File(getClass().getResource("/event-files").getFile()).getAbsolutePath();
+
+ Path outputPath = Files.createTempDirectory("output");
+
+ String[] args =
+ new String[] {"-f", eventFilePath, "-o", outputPath.toAbsolutePath().toString(), "-v"};
+
+ org.apache.gluten.qt.QualificationTool.main(args);
+
+ assert areFileContentsSame(
+ outputPath.resolve("AppsRecommendedForBoost.tsv"),
+ Paths.get(getClass().getResource("/AppsRecommendedForBoost.tsv").toURI()));
+ assert areFileContentsSame(
+ outputPath.resolve("UnsupportedOperators.tsv"),
+ Paths.get(getClass().getResource("/UnsupportedOperators.tsv").toURI()));
+ }
+}
diff --git a/tools/qualification-tool/src/test/resources/AppsRecommendedForBoost.tsv b/tools/qualification-tool/src/test/resources/AppsRecommendedForBoost.tsv
new file mode 100644
index 0000000000..41fd5104df
--- /dev/null
+++ b/tools/qualification-tool/src/test/resources/AppsRecommendedForBoost.tsv
@@ -0,0 +1,4 @@
+applicationId applicationName batchUuid rddPercentage unsupportedSqlPercentage totalTaskTime supportedTaskTime supportedSqlPercentage recommendedForBoost expectedRuntimeReduction
+local-1736931836972 QT - SQL UNKNOWN 0.0% 0.0% 111 111 100.0% true 30.0%
+local-1736931811258 QT - RDD UNKNOWN 100.0% 0.0% 22 0 0.0% false 0.0%
+local-1736932805054 QT - SQL 2 UNKNOWN 23.0% 0.0% 1167 0 0.0% false 0.0%
diff --git a/tools/qualification-tool/src/test/resources/UnsupportedOperators.tsv b/tools/qualification-tool/src/test/resources/UnsupportedOperators.tsv
new file mode 100644
index 0000000000..179311637c
--- /dev/null
+++ b/tools/qualification-tool/src/test/resources/UnsupportedOperators.tsv
@@ -0,0 +1,3 @@
+unsupportedOperator cumulativeCpuMs count
+Scan with format JSON not supported 0 4
+Execute CreateViewCommand not supported 0 2
diff --git a/tools/qualification-tool/src/test/resources/event-files/local-rdd b/tools/qualification-tool/src/test/resources/event-files/local-rdd
new file mode 100644
index 0000000000..7969e60b4a
--- /dev/null
+++ b/tools/qualification-tool/src/test/resources/event-files/local-rdd
@@ -0,0 +1,13 @@
+{"Event":"SparkListenerLogStart","Spark Version":"3.5.0"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
+{"Event":"SparkListenerExecutorAdded","Timestamp":1736931811308,"Executor ID":"driver","Executor Info":{"Host":"tim.c.googlers.com","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"tim.c.googlers.com","Port":37935},"Maximum Memory":18806420275,"Timestamp":1736931811331,"Maximum Onheap Memory":18806420275,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-openjdk-amd64","Java Version":"17.0.13 (Debian)","Scala Version":"version 2.12.15"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED [...]
+{"Event":"SparkListenerApplicationStart","App Name":"QT - RDD","App ID":"local-1736931811258","Timestamp":1736931810423,"User":"root"}
+{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1736931812150,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at QualificationToolTest.java:45","Number of Tasks":1,"RDD Info":[{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at QualificationToolTest.java:42","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1}, [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at QualificationToolTest.java:45","Number of Tasks":1,"RDD Info":[{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at QualificationToolTest.java:42","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"D [...]
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736931812284,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736931812284,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1736931812404,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Val [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at QualificationToolTest.java:45","Number of Tasks":1,"RDD Info":[{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at QualificationToolTest.java:42","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"D [...]
+{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1736931812418,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"SparkListenerApplicationEnd","Timestamp":1736931812428}
diff --git a/tools/qualification-tool/src/test/resources/event-files/local-sql-supported b/tools/qualification-tool/src/test/resources/event-files/local-sql-supported
new file mode 100644
index 0000000000..a8d6988273
--- /dev/null
+++ b/tools/qualification-tool/src/test/resources/event-files/local-sql-supported
@@ -0,0 +1,18 @@
+{"Event":"SparkListenerLogStart","Spark Version":"3.5.0"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
+{"Event":"SparkListenerExecutorAdded","Timestamp":1736931837027,"Executor ID":"driver","Executor Info":{"Host":"tim.c.googlers.com","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"tim.c.googlers.com","Port":38415},"Maximum Memory":18806420275,"Timestamp":1736931837043,"Maximum Onheap Memory":18806420275,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-openjdk-amd64","Java Version":"17.0.13 (Debian)","Scala Version":"version 2.12.15"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED [...]
+{"Event":"SparkListenerApplicationStart","App Name":"QT - SQL","App ID":"local-1736931836972","Timestamp":1736931836140,"User":"root"}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"createOrReplaceTempView at QualificationToolTest.java:36","details":"org.apache.spark.sql.Dataset.createOrReplaceTempView(Dataset.scala:3920)\norg.apache.gluten.qt.QualificationToolTest.testQualificationToolMain(QualificationToolTest.java:36)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMeth [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1736931839021,"errorMessage":""}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"rootExecutionId":1,"description":"show at QualificationToolTest.java:37","details":"org.apache.spark.sql.Dataset.show(Dataset.scala:807)\norg.apache.gluten.qt.QualificationToolTest.testQualificationToolMain(QualificationToolTest.java:37)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAcces [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[25,1],[26,4],[27,402]]}
+{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1736931839662,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"show at QualificationToolTest.java:37","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at QualificationToolTest.java:37","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Ba [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"show at QualificationToolTest.java:37","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at QualificationToolTest.java:37","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETE [...]
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736931839770,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736931839770,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1736931839984,"Failed":false,"Killed":false,"Accumulables":[{"ID":20,"Name":"duration","Update":"65","Value":"65","Internal":true,"Cou [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"show at QualificationToolTest.java:37","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at QualificationToolTest.java:37","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETE [...]
+{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1736931839995,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1736931840706,"errorMessage":""}
+{"Event":"SparkListenerApplicationEnd","Timestamp":1736931840714}
diff --git a/tools/qualification-tool/src/test/resources/event-files/local-unsupported b/tools/qualification-tool/src/test/resources/event-files/local-unsupported
new file mode 100644
index 0000000000..f59a7c10ba
--- /dev/null
+++ b/tools/qualification-tool/src/test/resources/event-files/local-unsupported
@@ -0,0 +1,24 @@
+{"Event":"SparkListenerLogStart","Spark Version":"3.5.0"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
+{"Event":"SparkListenerExecutorAdded","Timestamp":1736932805105,"Executor ID":"driver","Executor Info":{"Host":"tim.c.googlers.com","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"tim.c.googlers.com","Port":35139},"Maximum Memory":18806420275,"Timestamp":1736932805121,"Maximum Onheap Memory":18806420275,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-openjdk-amd64","Java Version":"17.0.13 (Debian)","Scala Version":"version 2.12.15"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED [...]
+{"Event":"SparkListenerApplicationStart","App Name":"QT - SQL 2","App ID":"local-1736932805054","Timestamp":1736932804170,"User":"root"}
+{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1736932808197,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"json at QualificationToolTest.java:46","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"json at QualificationToolTest.java:46","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":f [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"json at QualificationToolTest.java:46","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"json at QualificationToolTest.java:46","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE" [...]
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736932808317,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736932808317,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1736932808695,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"number of output rows","Update":"13","Value":"13","Interna [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"json at QualificationToolTest.java:46","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"json at QualificationToolTest.java:46","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE" [...]
+{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1736932808705,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"createOrReplaceTempView at QualificationToolTest.java:46","details":"org.apache.spark.sql.Dataset.createOrReplaceTempView(Dataset.scala:3920)\norg.apache.gluten.qt.QualificationToolTest.testQualificationToolMain(QualificationToolTest.java:46)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMeth [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1736932809049,"errorMessage":""}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"rootExecutionId":1,"description":"show at QualificationToolTest.java:48","details":"org.apache.spark.sql.Dataset.show(Dataset.scala:807)\norg.apache.gluten.qt.QualificationToolTest.testQualificationToolMain(QualificationToolTest.java:48)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAcces [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[61,1],[62,1],[63,100763]]}
+{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1736932809523,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"show at QualificationToolTest.java:48","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at QualificationToolTest.java:48","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Ba [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"show at QualificationToolTest.java:48","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at QualificationToolTest.java:48","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETE [...]
+{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736932809559,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1736932809559,"Executor ID":"driver","Host":"tim.c.googlers.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1736932810486,"Failed":false,"Killed":false,"Accumulables":[{"ID":59,"Name":"number of output rows","Update":"1","Value":"1","Internal [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"show at QualificationToolTest.java:48","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at QualificationToolTest.java:48","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETE [...]
+{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1736932810488,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1736932810521,"errorMessage":""}
+{"Event":"SparkListenerApplicationEnd","Timestamp":1736932810548}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org