You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2026/06/25 12:59:52 UTC
(spark) branch branch-4.x updated: [SPARK-57599][SQL] Decode Variant strings and keys as UTF-8
This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new a82e4480a1ea [SPARK-57599][SQL] Decode Variant strings and keys as UTF-8
a82e4480a1ea is described below
commit a82e4480a1ea13f4af16dfab5a22adcae491873f
Author: YangJie <ya...@baidu.com>
AuthorDate: Thu Jun 25 20:59:22 2026 +0800
[SPARK-57599][SQL] Decode Variant strings and keys as UTF-8
### What changes were proposed in this pull request?
`VariantUtil.getString` and `VariantUtil.getMetadataKey` decoded their bytes with `new String(bytes, offset, length)`, which uses the JVM default charset. This PR decodes both with `StandardCharsets.UTF_8`, matching how `VariantBuilder` writes keys and string values (`getBytes(StandardCharsets.UTF_8)`) and the UTF-8 string data defined by the variant binary format. A regression test, `VariantUtf8DecodeSuite`, is added in the `common/variant` module next to `VariantUtil`.
### Why are the changes needed?
The variant binary stores dictionary keys and string content as UTF-8, but the reader decoded them with the platform default charset, so the write and read sides disagree whenever that charset is not UTF-8. On Java 17 the default charset is environment-dependent (for example `US-ASCII` when the JVM starts under `LANG=C`), and reading a variant that contains non-ASCII keys or strings then silently corrupts those characters.
### Does this PR introduce _any_ user-facing change?
No change when the JVM default charset is UTF-8, which is the common case and what CI runs. Under a non-UTF-8 default charset, operations that read variant strings or keys -- `to_json(variant)`, casting a variant to string, `schema_of_variant`, and field/key extraction -- now return the correct characters instead of corrupting non-ASCII keys and string values.
### How was this patch tested?
Added `VariantUtf8DecodeSuite` in `common/variant`. A plain unit test cannot exercise the bug, because the default charset is fixed at JVM startup and pinned to `-Dfile.encoding=UTF-8` in CI, so the test forks a child JVM with `-Dfile.encoding=ISO-8859-1` and round-trips a variant with non-ASCII object keys and string values there, including a value over 63 UTF-8 bytes to cover the `LONG_STR` path. With the pre-fix default-charset decode the characters are corrupted (for example `café [...]
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 4.8)
Closes #56659 from LuciferYang/SPARK-variant-utf8-string-decode.
Authored-by: YangJie <ya...@baidu.com>
Signed-off-by: yangjie01 <ya...@baidu.com>
(cherry picked from commit 09205bbdeb9583a89dc868c6df98ce4295a4b97e)
Signed-off-by: yangjie01 <ya...@baidu.com>
---
.../apache/spark/types/variant/VariantUtil.java | 10 +-
.../types/variant/VariantUtf8DecodeSuite.scala | 118 +++++++++++++++++++++
2 files changed, 126 insertions(+), 2 deletions(-)
diff --git a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
index 1a3912775a2f..681c4038a9a9 100644
--- a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
+++ b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
@@ -26,6 +26,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.UUID;
@@ -515,7 +516,10 @@ public class VariantUtil {
length = readUnsigned(value, pos + 1, U32_SIZE);
}
checkIndex(start + length - 1, value.length);
- return new String(value, start, length);
+ // The string content is UTF-8 encoded (it is written by `VariantBuilder.appendString`).
+ // Decode with UTF-8 explicitly rather than relying on the JVM default charset, which is
+ // platform-dependent on Java 17.
+ return new String(value, start, length, StandardCharsets.UTF_8);
}
throw unexpectedType(Type.STRING);
}
@@ -701,6 +705,8 @@ public class VariantUtil {
int nextOffset = readUnsigned(metadata, 1 + (id + 2) * offsetSize, offsetSize);
if (offset > nextOffset) throw malformedVariant();
checkIndex(stringStart + nextOffset - 1, metadata.length);
- return new String(metadata, stringStart + offset, nextOffset - offset);
+ // Dictionary keys are UTF-8 encoded (see `VariantBuilder.addKey`). Decode with UTF-8
+ // explicitly rather than relying on the platform-dependent JVM default charset.
+ return new String(metadata, stringStart + offset, nextOffset - offset, StandardCharsets.UTF_8);
}
}
diff --git a/common/variant/src/test/scala/org/apache/spark/types/variant/VariantUtf8DecodeSuite.scala b/common/variant/src/test/scala/org/apache/spark/types/variant/VariantUtf8DecodeSuite.scala
new file mode 100644
index 000000000000..e196209c82fa
--- /dev/null
+++ b/common/variant/src/test/scala/org/apache/spark/types/variant/VariantUtf8DecodeSuite.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.types.variant
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Paths
+
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
+
+class VariantUtf8DecodeSuite extends AnyFunSuite { // scalastyle:ignore funsuite
+
+ test("SPARK-57599: keys and string values decode as UTF-8 under a non-UTF-8 default charset") {
+ // The reader must decode object keys and string values as UTF-8 regardless of the JVM default
+ // charset. That can only be exercised by changing the default charset, which is fixed at JVM
+ // startup and pinned to UTF-8 in the test JVM, so fork a child JVM with a non-UTF-8
+ // -Dfile.encoding and round-trip a variant with non-ASCII content there (see
+ // VariantUtf8DecodeChecker). With the pre-fix default-charset decode the characters are
+ // corrupted and the child exits non-zero.
+ val javaExe = Paths.get(sys.props("java.home"), "bin", "java").toString
+ val command = Seq(
+ javaExe,
+ "-Dfile.encoding=ISO-8859-1",
+ "-cp", sys.props("java.class.path"),
+ VariantUtf8DecodeChecker.getClass.getName.stripSuffix("$"))
+ val process = new ProcessBuilder(command: _*).redirectErrorStream(true).start()
+ val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8).trim
+ val exitCode = process.waitFor()
+ if (output.contains("RESULT=INCONCLUSIVE")) {
+ // The child JVM ended up with a UTF-8 default charset, so it cannot exercise the fix.
+ assume(false, s"Inconclusive; child output: $output")
+ }
+ assert(exitCode === 0, s"child JVM reported a decode failure; output: $output")
+ }
+}
+
+/**
+ * Entry point run in a child JVM by `VariantUtf8DecodeSuite` to verify that the Variant reader
+ * decodes object keys and string values as UTF-8 independently of the JVM default charset.
+ *
+ * This has to run as a separate `main`: the only way to actually exercise the bug is to start a JVM
+ * with a non-UTF-8 `-Dfile.encoding`, because the default charset is fixed at JVM startup and is
+ * pinned to UTF-8 in the test JVM. With the pre-fix default-charset decode, the non-ASCII content
+ * round-tripped here is corrupted and the process exits non-zero.
+ *
+ * All output is ASCII (characters are reported as code points) so it is readable regardless of the
+ * child's default charset. The process prints `RESULT=OK` and exits 0 on success, `RESULT=FAIL` and
+ * exits 1 if any value was corrupted, or `RESULT=INCONCLUSIVE` and exits 0 if the child's default
+ * charset turned out to be UTF-8 (in which case the fixed and buggy code cannot be told apart).
+ */
+private[variant] object VariantUtf8DecodeChecker {
+
+ // This object is a standalone process whose stdout is its result protocol, so println is used
+ // deliberately as the communication channel with the parent test.
+ // scalastyle:off println
+
+ // scalastyle:off nonascii
+ // (key, value) pairs covering 2-byte, 3-byte, and LONG_STR (> 63 UTF-8 bytes) UTF-8 content.
+ private val cases: Seq[(String, String)] = Seq(
+ "café" -> "résumé",
+ "你好" -> "世界",
+ "é" -> ("你" * 22)) // value is 66 UTF-8 bytes, over MAX_SHORT_STR_SIZE -> LONG_STR encoding
+ // scalastyle:on nonascii
+
+ def main(args: Array[String]): Unit = {
+ // Read the startup `file.encoding`, which determines the JVM default charset that the buggy
+ // `new String(bytes)` decode used. (Querying the default charset directly is banned by
+ // scalastyle.)
+ val fileEncoding = sys.props.getOrElse("file.encoding", "")
+ if (fileEncoding.equalsIgnoreCase("UTF-8")) {
+ // The fixed and buggy code are indistinguishable when the default charset is already UTF-8.
+ println(s"RESULT=INCONCLUSIVE fileEncoding=$fileEncoding")
+ System.exit(0)
+ }
+ var failures = 0
+ cases.foreach { case (key, value) =>
+ val json = "{" + jsonString(key) + ":" + jsonString(value) + "}"
+ val field = VariantBuilder.parseJson(json, false).getFieldAtIndex(0)
+ if (field.key != key) {
+ println(s"KEY_MISMATCH expected=${codePoints(key)} actual=${codePoints(field.key)}")
+ failures += 1
+ }
+ val decoded = field.value.getString
+ if (decoded != value) {
+ println(s"VALUE_MISMATCH expected=${codePoints(value)} actual=${codePoints(decoded)}")
+ failures += 1
+ }
+ }
+ if (failures == 0) {
+ println(s"RESULT=OK fileEncoding=$fileEncoding")
+ System.exit(0)
+ } else {
+ println(s"RESULT=FAIL failures=$failures fileEncoding=$fileEncoding")
+ System.exit(1)
+ }
+ }
+
+ private def jsonString(s: String): String = "\"" + s + "\""
+
+ private def codePoints(s: String): String =
+ s.codePoints().toArray.map(c => "U+%04X".format(c)).mkString("[", ",", "]")
+
+ // scalastyle:on println
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org