(spark) branch master updated: [SPARK-47327][SQL] Move sort keys concurrency test to CollationFactorySuite

[ma...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 6719168b6ec7 [SPARK-47327][SQL] Move sort keys concurrency test to CollationFactorySuite
6719168b6ec7 is described below

commit 6719168b6ec72242e111bcb3aae75985d36fdad2
Author: Stefan Kandic <st...@databricks.com>
AuthorDate: Sat Mar 16 09:24:22 2024 +0500

    [SPARK-47327][SQL] Move sort keys concurrency test to CollationFactorySuite
    
    ### What changes were proposed in this pull request?
    
    Move concurrency test to the `CollationFactorySuite`
    
    ### Why are the changes needed?
    
    This is more appropriate location for the test as it directly uses the `CollationFactory`.
    
    Also, I just found out that `par` method is highly discouraged and that we should use `ParSeq` instead.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No
    
    ### How was this patch tested?
    
    With existing UTs
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No
    
    Closes #45501 from stefankandic/moveTest.
    
    Authored-by: Stefan Kandic <st...@databricks.com>
    Signed-off-by: Max Gekk <ma...@gmail.com>
---
 common/unsafe/pom.xml                                      |  6 ++++++
 .../apache/spark/unsafe/types/CollationFactorySuite.scala  | 14 ++++++++++++++
 .../test/scala/org/apache/spark/sql/CollationSuite.scala   | 14 --------------
 3 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index e9785ebb7ad4..13b45f55a4ad 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -47,6 +47,12 @@
       <version>${project.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.scala-lang.modules</groupId>
+      <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>com.ibm.icu</groupId>
       <artifactId>icu4j</artifactId>
diff --git a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
index f9927b94fd42..0a9ff7558e3a 100644
--- a/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
+++ b/common/unsafe/src/test/scala/org/apache/spark/unsafe/types/CollationFactorySuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.unsafe.types
 
+import scala.collection.parallel.immutable.ParSeq
 import scala.jdk.CollectionConverters.MapHasAsScala
 
 import org.apache.spark.SparkException
@@ -138,4 +139,17 @@ class CollationFactorySuite extends AnyFunSuite with Matchers { // scalastyle:ig
       assert(result == testCase.expectedResult)
     })
   }
+
+  test("test concurrently generating collation keys") {
+    // generating ICU sort keys is not thread-safe by default so this should fail
+    // if we don't handle the concurrency properly on Collator level
+
+    (0 to 10).foreach(_ => {
+      val collator = fetchCollation("UNICODE").collator
+
+      ParSeq(0 to 100).foreach { _ =>
+        collator.getCollationKey("aaa")
+      }
+    })
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
index bef7417be36c..aaf3e88c9bdb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql
 
 import scala.collection.immutable.Seq
-import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable
 import scala.jdk.CollectionConverters.MapHasAsJava
 
 import org.apache.spark.SparkException
@@ -413,19 +412,6 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
     }
   }
 
-  test("test concurrently generating collation keys") {
-    // generating ICU sort keys is not thread-safe by default so this should fail
-    // if we don't handle the concurrency properly on Collator level
-
-    (0 to 10).foreach(_ => {
-      val collator = CollationFactory.fetchCollation("UNICODE").collator
-
-      (0 to 100).par.foreach { _ =>
-        collator.getCollationKey("aaa")
-      }
-    })
-  }
-
   test("text writing to parquet with collation enclosed with backticks") {
     withTempPath{ path =>
       sql(s"select 'a' COLLATE `UNICODE`").write.parquet(path.getAbsolutePath)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org