You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2024/10/18 13:01:38 UTC
(spark) branch master updated: [SPARK-49990][SQL] Improve performance of `randStr`
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7e3b1680b09f [SPARK-49990][SQL] Improve performance of `randStr`
7e3b1680b09f is described below
commit 7e3b1680b09f12edd37b21bdfab1e5916a2f64cb
Author: panbingkun <pa...@baidu.com>
AuthorDate: Fri Oct 18 15:01:22 2024 +0200
[SPARK-49990][SQL] Improve performance of `randStr`
### What changes were proposed in this pull request?
The pr aims to
- improve `performance` of `randStr`.
- make the `non-codegen` and `codegen` implementations of `randStr` as consistent as possible.
### Why are the changes needed?
Obviously, when creating `UTF8String`, using `UTF8String.fromBytes(bytes)` performs better than using `UTF8String.fromString(new String(bytes))`.
- `UTF8String.fromBytes(bytes)`
https://github.com/apache/spark/blob/39112e4f2f8c1401ffa73c84398d3b8f0afa211a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java#L160-L170
- `UTF8String.fromString(new String(bytes))`
https://github.com/apache/spark/blob/39112e4f2f8c1401ffa73c84398d3b8f0afa211a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java#L179-L184
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manually check.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48493 from panbingkun/SPARK-49990.
Authored-by: panbingkun <pa...@baidu.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../catalyst/expressions/ExpressionImplUtils.java | 19 ++++++++++++
.../catalyst/expressions/randomExpressions.scala | 35 ++--------------------
2 files changed, 22 insertions(+), 32 deletions(-)
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
index 18646f67975c..2fad36efe8cc 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
@@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.util.GenericArrayData;
import org.apache.spark.sql.errors.QueryExecutionErrors;
import org.apache.spark.unsafe.types.UTF8String;
import org.apache.spark.util.VersionUtils;
+import org.apache.spark.util.random.XORShiftRandom;
/**
* A utility class for constructing expressions.
@@ -315,4 +316,22 @@ public class ExpressionImplUtils {
}
return new GenericArrayData(res.toArray(new GenericArrayData[0]));
}
+
+ public static UTF8String randStr(XORShiftRandom rng, int length) {
+ byte[] bytes = new byte[length];
+ for (int i = 0; i < bytes.length; i++) {
+ // We generate a random number between 0 and 61, inclusive. Between the 62 different choices
+ // we choose 0-9, a-z, or A-Z, where each category comprises 10 choices, 26 choices, or 26
+ // choices, respectively (10 + 26 + 26 = 62).
+ int v = Math.abs(rng.nextInt() % 62);
+ if (v < 10) {
+ bytes[i] = (byte)('0' + v);
+ } else if (v < 36) {
+ bytes[i] = (byte)('a' + (v - 10));
+ } else {
+ bytes[i] = (byte)('A' + (v - 36));
+ }
+ }
+ return UTF8String.fromBytes(bytes);
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
index 706dc675d7f2..7148d3738f7f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.trees.{BinaryLike, TernaryLike, UnaryLike}
import org.apache.spark.sql.catalyst.trees.TreePattern.{EXPRESSION_WITH_RANDOM_SEED, RUNTIME_REPLACEABLE, TreePattern}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.random.XORShiftRandom
/**
@@ -399,23 +398,7 @@ case class RandStr(
override def evalInternal(input: InternalRow): Any = {
val numChars = length.eval(input).asInstanceOf[Number].intValue()
- val bytes = new Array[Byte](numChars)
- (0 until numChars).foreach { i =>
- // We generate a random number between 0 and 61, inclusive. Between the 62 different choices
- // we choose 0-9, a-z, or A-Z, where each category comprises 10 choices, 26 choices, or 26
- // choices, respectively (10 + 26 + 26 = 62).
- val num = (rng.nextInt() % 62).abs
- num match {
- case _ if num < 10 =>
- bytes.update(i, ('0' + num).toByte)
- case _ if num < 36 =>
- bytes.update(i, ('a' + num - 10).toByte)
- case _ =>
- bytes.update(i, ('A' + num - 36).toByte)
- }
- }
- val result: UTF8String = UTF8String.fromBytes(bytes.toArray)
- result
+ ExpressionImplUtils.randStr(rng, numChars)
}
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -427,19 +410,8 @@ case class RandStr(
ev.copy(code =
code"""
|${eval.code}
- |int length = (int)(${eval.value});
- |char[] chars = new char[length];
- |for (int i = 0; i < length; i++) {
- | int v = Math.abs($rngTerm.nextInt() % 62);
- | if (v < 10) {
- | chars[i] = (char)('0' + v);
- | } else if (v < 36) {
- | chars[i] = (char)('a' + (v - 10));
- | } else {
- | chars[i] = (char)('A' + (v - 36));
- | }
- |}
- |UTF8String ${ev.value} = UTF8String.fromString(new String(chars));
+ |UTF8String ${ev.value} =
+ | ${classOf[ExpressionImplUtils].getName}.randStr($rngTerm, (int)(${eval.value}));
|boolean ${ev.isNull} = false;
|""".stripMargin,
isNull = FalseLiteral)
@@ -452,4 +424,3 @@ object RandStr {
def apply(length: Expression, seedExpression: Expression): RandStr =
RandStr(length, seedExpression, hideSeed = false)
}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org