You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ma...@apache.org on 2026/06/29 10:06:21 UTC
(gluten) branch main updated: [GLUTEN-11539][VL] Improve error message for unsupported spark.io.compression.codec (#12360)
This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3a36ef2357 [GLUTEN-11539][VL] Improve error message for unsupported spark.io.compression.codec (#12360)
3a36ef2357 is described below
commit 3a36ef235727cdb05a20a7fd64bb6b93f66f19b6
Author: BRIJ RAJ KISHORE <22...@users.noreply.github.com>
AuthorDate: Mon Jun 29 15:36:15 2026 +0530
[GLUTEN-11539][VL] Improve error message for unsupported spark.io.compression.codec (#12360)
Emit a unified error for all unsupported codecs that names the
offending codec, hints at spark.shuffle.compress=false to disable
compression entirely, and lists the valid codec choices.
Co-authored-by: Claude Sonnet 4.6 <no...@anthropic.com>
---
.../test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala | 7 ++++---
.../main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala | 6 +++---
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index fcf1fbbbf7..51cb918aa6 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -2257,16 +2257,17 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa
val ex = intercept[IllegalArgumentException] {
GlutenShuffleUtils.getCompressionCodec(conf)
}
- assert(ex.getMessage.contains("snappy is not supported"))
+ assert(ex.getMessage.contains("does not support codec 'snappy'"))
+ assert(ex.getMessage.contains("spark.shuffle.compress=false"))
assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key))
}
- test("GLUTEN-11539: spark.io.compression.codec=none throws with actionable message") {
+ test("GLUTEN-11539: spark.io.compression.codec=none throws pointing to spark.shuffle.compress") {
val conf = spark.sparkContext.getConf.clone().set("spark.io.compression.codec", "none")
val ex = intercept[IllegalArgumentException] {
GlutenShuffleUtils.getCompressionCodec(conf)
}
- assert(ex.getMessage.contains("none is not supported"))
+ assert(ex.getMessage.contains("spark.shuffle.compress=false"))
assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key))
}
diff --git a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
index 4bd89582c1..d7d8d4cbd9 100644
--- a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
+++ b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
@@ -77,9 +77,9 @@ object GlutenShuffleUtils {
val supportedCodecs = BackendsApiManager.getSettings.shuffleSupportedCodec()
if (!supportedCodecs.contains(codec)) {
throw new IllegalArgumentException(
- s"Gluten shuffle only supports ${supportedCodecs.mkString(", ")}. " +
- s"$codec is not supported. " +
- s"You may configure ${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} " +
+ s"Gluten shuffle does not support codec '$codec'. " +
+ s"To disable shuffle compression, set spark.shuffle.compress=false. " +
+ s"To use a supported codec, set ${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} " +
s"to ${supportedCodecs.mkString(" or ")}.")
}
codec
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org