You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/07/27 00:27:00 UTC
[spark] branch master updated: [SPARK-39879][SQL][TESTS] Reduce local-cluster maximum memory size in `BroadcastJoinSuite*` and `HiveSparkSubmitSuite`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 01d41e7de41 [SPARK-39879][SQL][TESTS] Reduce local-cluster maximum memory size in `BroadcastJoinSuite*` and `HiveSparkSubmitSuite`
01d41e7de41 is described below
commit 01d41e7de418d0a40db7b16ddd0d8546f0794d17
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Wed Jul 27 09:26:47 2022 +0900
[SPARK-39879][SQL][TESTS] Reduce local-cluster maximum memory size in `BroadcastJoinSuite*` and `HiveSparkSubmitSuite`
### What changes were proposed in this pull request?
This pr change `local-cluster[2, 1, 1024]` in `BroadcastJoinSuite*` and `HiveSparkSubmitSuite` to `local-cluster[2, 1, 512]` to reduce test maximum memory usage.
### Why are the changes needed?
Reduce the maximum memory usage of test cases.
### Does this PR introduce _any_ user-facing change?
No, test-only.
### How was this patch tested?
Should monitor CI
Closes #37298 from LuciferYang/reduce-local-cluster-memory.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../sql/execution/joins/BroadcastJoinSuite.scala | 4 +-
.../spark/sql/hive/HiveSparkSubmitSuite.scala | 43 +++++++++++++++-------
2 files changed, 32 insertions(+), 15 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 3cc43e2dd41..6333808b420 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.joins
import scala.reflect.ClassTag
import org.apache.spark.AccumulatorSuite
+import org.apache.spark.internal.config.EXECUTOR_MEMORY
import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession}
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, Cast, Expression, Literal, ShiftLeft}
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
@@ -56,7 +57,8 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
override def beforeAll(): Unit = {
super.beforeAll()
spark = SparkSession.builder()
- .master("local-cluster[2,1,1024]")
+ .master("local-cluster[2,1,512]")
+ .config(EXECUTOR_MEMORY.key, "512m")
.appName("testing")
.getOrCreate()
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 18202307fc4..fa4d1b78d1c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -33,6 +33,7 @@ import org.scalatest.time.SpanSugar._
import org.apache.spark._
import org.apache.spark.deploy.SparkSubmitTestUtils
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.EXECUTOR_MEMORY
import org.apache.spark.internal.config.UI.UI_ENABLED
import org.apache.spark.sql.{QueryTest, Row, SparkSession}
import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
@@ -74,7 +75,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", TemporaryHiveUDFTest.getClass.getName.stripSuffix("$"),
"--name", "TemporaryHiveUDFTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -91,7 +93,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", PermanentHiveUDFTest1.getClass.getName.stripSuffix("$"),
"--name", "PermanentHiveUDFTest1",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -108,7 +111,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", PermanentHiveUDFTest2.getClass.getName.stripSuffix("$"),
"--name", "PermanentHiveUDFTest2",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -127,7 +131,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SparkSubmitClassLoaderTest.getClass.getName.stripSuffix("$"),
"--name", "SparkSubmitClassLoaderTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -142,7 +147,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SparkSQLConfTest.getClass.getName.stripSuffix("$"),
"--name", "SparkSQLConfTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--conf", "spark.sql.hive.metastore.version=0.12",
@@ -180,7 +186,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SPARK_9757.getClass.getName.stripSuffix("$"),
"--name", "SparkSQLConfTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -193,7 +200,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SPARK_11009.getClass.getName.stripSuffix("$"),
"--name", "SparkSQLConfTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -206,7 +214,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SPARK_14244.getClass.getName.stripSuffix("$"),
"--name", "SparkSQLConfTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -219,7 +228,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SetWarehouseLocationTest.getClass.getName.stripSuffix("$"),
"--name", "SetSparkWarehouseLocationTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -256,7 +266,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SetWarehouseLocationTest.getClass.getName.stripSuffix("$"),
"--name", "SetHiveWarehouseLocationTest",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--conf", s"spark.sql.test.expectedWarehouseDir=$hiveWarehouseLocation",
@@ -314,7 +325,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SPARK_18360.getClass.getName.stripSuffix("$"),
"--name", "SPARK-18360",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--driver-java-options", "-Dderby.system.durability=test",
@@ -328,7 +340,8 @@ class HiveSparkSubmitSuite
val argsForCreateTable = Seq(
"--class", SPARK_18989_CREATE_TABLE.getClass.getName.stripSuffix("$"),
"--name", "SPARK-18947",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--jars", HiveTestJars.getHiveContribJar().getCanonicalPath,
@@ -338,7 +351,8 @@ class HiveSparkSubmitSuite
val argsForShowTables = Seq(
"--class", SPARK_18989_DESC_TABLE.getClass.getName.stripSuffix("$"),
"--name", "SPARK-18947",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
unusedJar.toString)
@@ -359,7 +373,8 @@ class HiveSparkSubmitSuite
val args = Seq(
"--class", SPARK_34772.getClass.getName.stripSuffix("$"),
"--name", "SPARK-34772",
- "--master", "local-cluster[2,1,1024]",
+ "--master", "local-cluster[2,1,512]",
+ "--conf", s"${EXECUTOR_MEMORY.key}=512m",
"--conf", s"${LEGACY_TIME_PARSER_POLICY.key}=LEGACY",
"--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=1.2.1",
"--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org