You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2021/11/20 02:31:55 UTC
[spark] branch master updated: [SPARK-37282][TESTS][FOLLOWUP] Extract `Utils.isMacOnAppleSilicon` for reuse in UTs
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c6c72a4 [SPARK-37282][TESTS][FOLLOWUP] Extract `Utils.isMacOnAppleSilicon` for reuse in UTs
c6c72a4 is described below
commit c6c72a453b9958c32a16be3f199e256dc7bc17ae
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Fri Nov 19 18:31:00 2021 -0800
[SPARK-37282][TESTS][FOLLOWUP] Extract `Utils.isMacOnAppleSilicon` for reuse in UTs
### What changes were proposed in this pull request?
SPARK-37282 use code similar to `SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64")` to determine whether the current test environment is `isMacOnAppleSilicon`. This pr extracted this duplicate code to `Utils.isMacOnAppleSilicon` and reuse it in UTs.
### Why are the changes needed?
Remove duplicate codes.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass the Jenkins or GitHub Action
- Manual test changed UTs use Apple Silicon passed
Closes #34648 from LuciferYang/SPARK-37282-FOLLOWUP.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../java/org/apache/spark/util/kvstore/LevelDBIteratorSuite.java | 2 +-
.../src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java | 2 +-
core/src/main/scala/org/apache/spark/util/Utils.scala | 5 +++++
.../org/apache/spark/deploy/history/FsHistoryProviderSuite.scala | 2 +-
.../scala/org/apache/spark/deploy/history/HistoryServerSuite.scala | 4 ++--
.../src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala | 2 +-
.../org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala | 2 +-
7 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBIteratorSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBIteratorSuite.java
index ea814cb..ceab771 100644
--- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBIteratorSuite.java
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBIteratorSuite.java
@@ -41,7 +41,7 @@ public class LevelDBIteratorSuite extends DBIteratorSuite {
@Override
protected KVStore createStore() throws Exception {
- assumeFalse(SystemUtils.IS_OS_MAC_OSX && System.getProperty("os.arch").equals("aarch64"));
+ assumeFalse(SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64"));
dbpath = File.createTempFile("test.", ".ldb");
dbpath.delete();
db = new LevelDB(dbpath);
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java
index 1134ec2..ef92a6c 100644
--- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/LevelDBSuite.java
@@ -52,7 +52,7 @@ public class LevelDBSuite {
@Before
public void setup() throws Exception {
- assumeFalse(SystemUtils.IS_OS_MAC_OSX && System.getProperty("os.arch").equals("aarch64"));
+ assumeFalse(SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64"));
dbpath = File.createTempFile("test.", ".ldb");
dbpath.delete();
db = new LevelDB(dbpath);
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 0029bbd..27496d6 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1962,6 +1962,11 @@ private[spark] object Utils extends Logging {
val isMac = SystemUtils.IS_OS_MAC_OSX
/**
+ * Whether the underlying operating system is Mac OS X and processor is Apple Silicon.
+ */
+ val isMacOnAppleSilicon = SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64")
+
+ /**
* Pattern for matching a Windows drive, which contains only a single alphabet character.
*/
val windowsDrive = "([a-zA-Z])".r
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index c3d524e..b05b9de 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -1652,7 +1652,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
if (!inMemory) {
// LevelDB doesn't support Apple Silicon yet
- assume(!(Utils.isMac && System.getProperty("os.arch").equals("aarch64")))
+ assume(!Utils.isMacOnAppleSilicon)
conf.set(LOCAL_STORE_DIR, Utils.createTempDir().getAbsolutePath())
}
conf.set(HYBRID_STORE_ENABLED, useHybridStore)
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index a9892b1..0f4481e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -87,7 +87,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
.set(EXECUTOR_PROCESS_TREE_METRICS_ENABLED, true)
conf.setAll(extraConf)
// Since LevelDB doesn't support Apple Silicon yet, fallback to in-memory provider
- if (Utils.isMac && System.getProperty("os.arch").equals("aarch64")) {
+ if (Utils.isMacOnAppleSilicon) {
conf.remove(LOCAL_STORE_DIR)
}
provider = new FsHistoryProvider(conf)
@@ -389,7 +389,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
.set(LOCAL_STORE_DIR, storeDir.getAbsolutePath())
.remove(IS_TESTING)
// Since LevelDB doesn't support Apple Silicon yet, fallback to in-memory provider
- if (Utils.isMac && System.getProperty("os.arch").equals("aarch64")) {
+ if (Utils.isMacOnAppleSilicon) {
myConf.remove(LOCAL_STORE_DIR)
}
val provider = new FsHistoryProvider(myConf)
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
index 0b32c7f..d7fefb6 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
@@ -84,7 +84,7 @@ class AppStatusStoreSuite extends SparkFunSuite {
return AppStatusStore.createLiveStore(conf)
}
// LevelDB doesn't support Apple Silicon yet
- if (Utils.isMac && System.getProperty("os.arch").equals("aarch64") && disk) {
+ if (Utils.isMacOnAppleSilicon && disk) {
return null
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala
index ed20f25..e82b9df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala
@@ -51,7 +51,7 @@ class StreamingSessionWindowSuite extends StreamTest
(SQLConf.STATE_STORE_PROVIDER_CLASS.key, value.stripSuffix("$"))
}
// RocksDB doesn't support Apple Silicon yet
- if (Utils.isMac && System.getProperty("os.arch").equals("aarch64")) {
+ if (Utils.isMacOnAppleSilicon) {
providerOptions = providerOptions
.filterNot(_._2.contains(classOf[RocksDBStateStoreProvider].getSimpleName))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org