You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/08/07 20:01:21 UTC
(spark) branch master updated: [SPARK-53179][CORE][TESTS] Use `SparkStreamUtils.toString` instead of `CharStreams.toString`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 106841389239 [SPARK-53179][CORE][TESTS] Use `SparkStreamUtils.toString` instead of `CharStreams.toString`
106841389239 is described below
commit 1068413892393c02b4ed0629d8434b02f49bd36e
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Aug 7 13:01:08 2025 -0700
[SPARK-53179][CORE][TESTS] Use `SparkStreamUtils.toString` instead of `CharStreams.toString`
### What changes were proposed in this pull request?
This PR aims to se `SparkStreamUtils.toString` instead of `CharStreams.toString`.
### Why are the changes needed?
`SparkStreamUtils.toString` is ***faster*** than `CharStreams.toString`.
```scala
scala> spark.time(org.apache.spark.util.SparkStreamUtils.toString(new java.io.FileInputStream("/tmp/1G.bin")).length)
Time taken: 322 ms
val res0: Int = 1073741824
scala> spark.time(com.google.common.io.CharStreams.toString(new java.io.InputStreamReader(java.nio.file.Files.newInputStream(Path.of("/tmp/1G.bin")))).length)
Time taken: 533 ms
val res1: Int = 1073741824
```
### Does this PR introduce _any_ user-facing change?
No behavior change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #51908 from dongjoon-hyun/SPARK-53179.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../shuffle/ExternalShuffleBlockResolverSuite.java | 15 ++++-----------
.../spark/deploy/ExternalShuffleServiceDbSuite.scala | 5 +----
.../network/netty/NettyBlockTransferSecuritySuite.scala | 8 ++------
dev/checkstyle.xml | 4 ++++
scalastyle-config.xml | 5 +++++
5 files changed, 16 insertions(+), 21 deletions(-)
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
index 311827dbed4c..3e068d457368 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
@@ -19,15 +19,14 @@ package org.apache.spark.network.shuffle;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.io.CharStreams;
import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
import org.apache.spark.network.util.MapConfigProvider;
import org.apache.spark.network.util.TransportConf;
import org.apache.spark.network.shuffle.ExternalShuffleBlockResolver.AppExecId;
+import org.apache.spark.util.SparkStreamUtils$;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -83,23 +82,17 @@ public class ExternalShuffleBlockResolverSuite {
try (InputStream block0Stream = resolver.getBlockData(
"app0", "exec0", 0, 0, 0).createInputStream()) {
- String block0 =
- CharStreams.toString(new InputStreamReader(block0Stream, StandardCharsets.UTF_8));
- assertEquals(sortBlock0, block0);
+ assertEquals(sortBlock0, SparkStreamUtils$.MODULE$.toString(block0Stream));
}
try (InputStream block1Stream = resolver.getBlockData(
"app0", "exec0", 0, 0, 1).createInputStream()) {
- String block1 =
- CharStreams.toString(new InputStreamReader(block1Stream, StandardCharsets.UTF_8));
- assertEquals(sortBlock1, block1);
+ assertEquals(sortBlock1, SparkStreamUtils$.MODULE$.toString(block1Stream));
}
try (InputStream blocksStream = resolver.getContinuousBlocksData(
"app0", "exec0", 0, 0, 0, 2).createInputStream()) {
- String blocks =
- CharStreams.toString(new InputStreamReader(blocksStream, StandardCharsets.UTF_8));
- assertEquals(sortBlock0 + sortBlock1, blocks);
+ assertEquals(sortBlock0 + sortBlock1, SparkStreamUtils$.MODULE$.toString(blocksStream));
}
}
diff --git a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceDbSuite.scala b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceDbSuite.scala
index 921175bd4103..224f47f1d89c 100644
--- a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceDbSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceDbSuite.scala
@@ -17,11 +17,8 @@
package org.apache.spark.deploy
-import java.io._
import java.nio.charset.StandardCharsets
-import com.google.common.io.CharStreams
-
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.internal.config._
import org.apache.spark.network.shuffle.{ExternalBlockHandler, ExternalShuffleBlockResolver}
@@ -110,7 +107,7 @@ abstract class ExternalShuffleServiceDbSuite extends SparkFunSuite {
blockResolver = blockHandler.getBlockResolver
val block0Stream = blockResolver.getBlockData("app0", "exec0", 0, 0, 0).createInputStream
- val block0 = CharStreams.toString(new InputStreamReader(block0Stream, StandardCharsets.UTF_8))
+ val block0 = Utils.toString(block0Stream)
block0Stream.close()
assert(sortBlock0 == block0)
// pass
diff --git a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala
index 124c089b4710..2733800727c0 100644
--- a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala
+++ b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferSecuritySuite.scala
@@ -17,7 +17,6 @@
package org.apache.spark.network.netty
-import java.io.InputStreamReader
import java.nio._
import java.nio.charset.StandardCharsets
import java.util.concurrent.TimeUnit
@@ -26,7 +25,6 @@ import scala.concurrent.Promise
import scala.concurrent.duration._
import scala.util.{Failure, Success, Try}
-import com.google.common.io.CharStreams
import org.mockito.Mockito._
import org.scalatest.matchers.must.Matchers
import org.scalatest.matchers.should.Matchers._
@@ -40,7 +38,7 @@ import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.shuffle.BlockFetchingListener
import org.apache.spark.serializer.{JavaSerializer, SerializerManager}
import org.apache.spark.storage.{BlockId, ShuffleBlockId}
-import org.apache.spark.util.{SslTestUtils, ThreadUtils}
+import org.apache.spark.util.{SslTestUtils, ThreadUtils, Utils}
class NettyBlockTransferSecuritySuite extends SparkFunSuite with MockitoSugar with Matchers {
@@ -150,9 +148,7 @@ class NettyBlockTransferSecuritySuite extends SparkFunSuite with MockitoSugar wi
val result = fetchBlock(exec0, exec1, "1", blockId) match {
case Success(buf) =>
- val actualString = CharStreams.toString(
- new InputStreamReader(buf.createInputStream(), StandardCharsets.UTF_8))
- actualString should equal(blockString)
+ Utils.toString(buf.createInputStream()) should equal(blockString)
buf.release()
Success(())
case Failure(t) =>
diff --git a/dev/checkstyle.xml b/dev/checkstyle.xml
index 7d34afc99345..f9f08ed124fc 100644
--- a/dev/checkstyle.xml
+++ b/dev/checkstyle.xml
@@ -229,6 +229,10 @@
<property name="format" value="FileUtils\.deleteQuietly"/>
<property name="message" value="Use deleteQuietly of JavaUtils/SparkFileUtils/Utils instead." />
</module>
+ <module name="RegexpSinglelineJava">
+ <property name="format" value="CharStreams\.toString"/>
+ <property name="message" value="Use toString of SparkStreamUtils or Utils instead." />
+ </module>
<!-- support structured logging -->
<module name="RegexpSinglelineJava">
<property name="format" value="org\.slf4j\.(Logger|LoggerFactory)" />
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index 75794f0bbc1a..abb771a78d37 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -722,6 +722,11 @@ This file is divided into 3 sections:
<customMessage>Use toString of SparkStreamUtils or Utils instead.</customMessage>
</check>
+ <check customId="charstreamstostring" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
+ <parameters><parameter name="regex">\bCharStreams\.toString\b</parameter></parameters>
+ <customMessage>Use toString of SparkStreamUtils or Utils instead.</customMessage>
+ </check>
+
<check customId="ioutilswrite" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
<parameters><parameter name="regex">\bIOUtils\.write\b</parameter></parameters>
<customMessage>Use Java `write` instead.</customMessage>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org