You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/04/28 19:07:19 UTC
(spark) branch master updated: [SPARK-51094][SQL][TESTS] Adjust some test parameters to work on s390x
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fb2705c796ff [SPARK-51094][SQL][TESTS] Adjust some test parameters to work on s390x
fb2705c796ff is described below
commit fb2705c796ff64de147fa06f83fe05dbcd591af4
Author: Jonathan Albrecht <jo...@ibm.com>
AuthorDate: Mon Apr 28 12:07:03 2025 -0700
[SPARK-51094][SQL][TESTS] Adjust some test parameters to work on s390x
### What changes were proposed in this pull request?
Adjust some test parameters to work on s390x
CoalesceShufflePartitionsSuite: The default lz4 compression does not generate the same compressed stream on all architectures especially for different endianness. Increase the maxRange on big endian so the same number of partitions are generated as on little endian.
AdaptiveQueryExecSuite: Change the ADVISORY_PARTITION_SIZE_IN_BYTES from 1000 to 900 so that the same number of partitions is generated on amd64 and s390x.
ClientE2ETestSuite: Change the recursion depth from 500 to 400 for s390x only since on that platform the jvm hits the maximum recursion depth sooner.
### Why are the changes needed?
To allow tests to pass on s390x
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Ran existing tests on amd64 (little endian) and s390x (big endian)
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #49813 from jonathan-albrecht-ibm/master-endian-testParams.
Authored-by: Jonathan Albrecht <jo...@ibm.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../org/apache/spark/sql/connect/ClientE2ETestSuite.scala | 5 +++--
.../sql/execution/CoalesceShufflePartitionsSuite.scala | 13 ++++++++++---
.../sql/execution/adaptive/AdaptiveQueryExecSuite.scala | 2 +-
3 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
index 566584f7e7a8..415960e83f9d 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
@@ -228,11 +228,12 @@ class ClientE2ETestSuite
}
test("spark deep recursion") {
+ var recursionDepth = if (System.getProperty("os.arch") == "s390x") 400 else 500
var df = spark.range(1)
- for (a <- 1 to 500) {
+ for (a <- 1 to recursionDepth) {
df = df.union(spark.range(a, a + 1))
}
- assert(df.collect().length == 501)
+ assert(df.collect().length == recursionDepth + 1)
}
test("handle unknown exception") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
index 0a06e3cc3f6e..4b650957e42c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.execution
+import java.nio.ByteOrder.{nativeOrder, BIG_ENDIAN}
+
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.internal.config.IO_ENCRYPTION_ENABLED
import org.apache.spark.internal.config.UI.UI_ENABLED
@@ -176,16 +178,21 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with SQLConfHelper
test(s"determining the number of reducers: complex query 1$testNameNote") {
val test: (SparkSession) => Unit = { spark: SparkSession =>
+ // The default lz4 compression does not generate the same compressed
+ // stream on all architectures especially for different endianness.
+ // Increase the maxRange on big endian so the same number of partitions
+ // are generated as on little endian.
+ val maxRange = if (nativeOrder().equals(BIG_ENDIAN)) 2000 else 1000
val df1 =
spark
- .range(0, 1000, 1, numInputPartitions)
+ .range(0, maxRange, 1, numInputPartitions)
.selectExpr("id % 500 as key1", "id as value1")
.groupBy("key1")
.count()
.toDF("key1", "cnt1")
val df2 =
spark
- .range(0, 1000, 1, numInputPartitions)
+ .range(0, maxRange, 1, numInputPartitions)
.selectExpr("id % 500 as key2", "id as value2")
.groupBy("key2")
.count()
@@ -197,7 +204,7 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite with SQLConfHelper
val expectedAnswer =
spark
.range(0, 500)
- .selectExpr("id", "2 as cnt")
+ .selectExpr("id", s"${maxRange/500}L as cnt")
QueryTest.checkAnswer(
join,
expectedAnswer.collect().toImmutableArraySeq)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 8c46b6d8b6d4..13de81065cb7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -1127,7 +1127,7 @@ class AdaptiveQueryExecSuite
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
SQLConf.SHUFFLE_PARTITIONS.key -> "100",
SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800",
- SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "1000") {
+ SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "900") {
withTempView("skewData1", "skewData2") {
spark
.range(0, 1000, 1, 10)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org