You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by su...@apache.org on 2024/02/24 16:23:33 UTC
(arrow-datafusion-comet) branch main updated: build: Add CI for TPCDS queries (#99)
This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 38b0bfd build: Add CI for TPCDS queries (#99)
38b0bfd is described below
commit 38b0bfdd80b90a26919ee010778b4f3a3b58249c
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Sat Feb 24 08:23:29 2024 -0800
build: Add CI for TPCDS queries (#99)
---
.github/workflows/pr_build.yml | 61 ++++++++++++++++++++++
pom.xml | 1 +
.../apache/spark/sql/CometTPCDSQuerySuite.scala | 3 +-
3 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
index 55117d3..669eddd 100644
--- a/.github/workflows/pr_build.yml
+++ b/.github/workflows/pr_build.yml
@@ -131,3 +131,64 @@ jobs:
- if: matrix.test-target == 'java'
name: Java test steps
uses: ./.github/actions/java-test
+
+ tpcds-1g:
+ name: Run TPC-DS queries with SF=1
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ env:
+ JAVA_VERSION: 11
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Rust & Java toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: ${{env.RUST_VERSION}}
+ jdk-version: 11
+
+ - name: Cache TPC-DS generated data
+ id: cache-tpcds-sf-1
+ uses: actions/cache@v4
+ with:
+ path: ./tpcds-sf-1
+ key: tpcds-${{ hashFiles('.github/workflows/pr_build.yml') }}
+ - name: Checkout tpcds-kit repository
+ if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+ uses: actions/checkout@v4
+ with:
+ repository: databricks/tpcds-kit
+ path: ./tpcds-kit
+ - name: Build Comet
+ run: make release
+ - name: Build tpcds-kit
+ if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+ run: |
+ apt-get install -y yacc bison flex
+ cd tpcds-kit/tools && make OS=LINUX
+ - name: Generate TPC-DS (SF=1) table data
+ if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+ run: |
+ cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location `pwd`/../tpcds-sf-1 --scaleFactor 1 --numPartitions 1"
+ cd ..
+ - name: Run TPC-DS queries (Sort merge join)
+ run: |
+ SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
+ env:
+ SPARK_TPCDS_JOIN_CONF: |
+ spark.sql.autoBroadcastJoinThreshold=-1
+ spark.sql.join.preferSortMergeJoin=true
+ - name: Run TPC-DS queries (Broadcast hash join)
+ run: |
+ SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
+ env:
+ SPARK_TPCDS_JOIN_CONF: |
+ spark.sql.autoBroadcastJoinThreshold=10485760
+ - name: Run TPC-DS queries (Shuffled hash join)
+ run: |
+ SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
+ env:
+ SPARK_TPCDS_JOIN_CONF: |
+ spark.sql.autoBroadcastJoinThreshold=-1
+ spark.sql.join.forceApplyShuffledHashJoin=true
+
diff --git a/pom.xml b/pom.xml
index 507ec78..5cac3c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -835,6 +835,7 @@ under the License.
<exclude>**/test/resources/**</exclude>
<exclude>**/benchmarks/*.txt</exclude>
<exclude>**/inspections/*.txt</exclude>
+ <exclude>tpcds-kit/**</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala
index 443c1cb..5d7b603 100644
--- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala
@@ -26,7 +26,8 @@ import org.apache.comet.CometConf
class CometTPCDSQuerySuite
extends {
// This is private in `TPCDSBase`.
- val excludedTpcdsQueries: Seq[String] = Seq("q34", "q64")
+ val excludedTpcdsQueries: Seq[String] =
+ Seq("q34", "q66", "q64", "q71", "q88", "q90", "q96")
// This is private in `TPCDSBase` and `excludedTpcdsQueries` is private too.
// So we cannot override `excludedTpcdsQueries` to exclude the queries.