You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by vi...@apache.org on 2021/08/05 08:33:01 UTC
[spark] branch branch-3.2 updated: [SPARK-36393][BUILD] Try to
raise memory for GHA
This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 712c311 [SPARK-36393][BUILD] Try to raise memory for GHA
712c311 is described below
commit 712c31173625ddc03e3ca8428909ae26f1491625
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Thu Aug 5 01:31:35 2021 -0700
[SPARK-36393][BUILD] Try to raise memory for GHA
### What changes were proposed in this pull request?
According to the feedback from GitHub, the change causing memory issue has been rolled back. We can try to raise memory again for GA.
### Why are the changes needed?
Trying higher memory settings for GA. It could speed up the testing time.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
GA
Closes #33623 from viirya/increasing-mem-ga.
Authored-by: Liang-Chi Hsieh <vi...@gmail.com>
Signed-off-by: Liang-Chi Hsieh <vi...@gmail.com>
(cherry picked from commit 7d13ac177ba745e42c406ee8aa1594cc3448dbc6)
Signed-off-by: Liang-Chi Hsieh <vi...@gmail.com>
---
.github/workflows/build_and_test.yml | 2 +-
build/sbt-launch-lib.bash | 6 ++++--
dev/run-tests.py | 7 +------
pom.xml | 8 ++++----
4 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 3eb12f5..1d3d9c5 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -170,7 +170,7 @@ jobs:
SPARK_LOCAL_IP: localhost
SKIP_UNIDOC: true
SKIP_MIMA: true
- METASPACE_SIZE: 128m
+ METASPACE_SIZE: 1g
steps:
- name: Checkout Spark repository
uses: actions/checkout@v2
diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash
index 7261e01..8fb6672 100755
--- a/build/sbt-launch-lib.bash
+++ b/build/sbt-launch-lib.bash
@@ -117,9 +117,11 @@ addDebugger () {
# so they need not be dicked around with individually.
get_mem_opts () {
local mem=${1:-$sbt_default_mem}
- local codecache=128
+ local codecache=$(( $mem / 8 ))
+ (( $codecache > 128 )) || codecache=128
+ (( $codecache < 2048 )) || codecache=2048
- echo "-Xms256m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m"
+ echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m"
}
require_arg () {
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 4828bba..c54042c 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -268,12 +268,7 @@ def exec_sbt(sbt_args=()):
"""Will call SBT in the current directory with the list of mvn_args passed
in and returns the subprocess for any further processing"""
- sbt_cmd = [os.path.join(SPARK_HOME, "build", "sbt")]
-
- if "GITHUB_ACTIONS" in os.environ:
- sbt_cmd = sbt_cmd + ['-mem', '2300']
-
- sbt_cmd = sbt_cmd + sbt_args
+ sbt_cmd = [os.path.join(SPARK_HOME, "build", "sbt")] + sbt_args
sbt_output_filter = re.compile(b"^.*[info].*Resolving" + b"|" +
b"^.*[warn].*Merging" + b"|" +
diff --git a/pom.xml b/pom.xml
index 3d2548f..954b703c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2611,8 +2611,8 @@
</args>
<jvmArgs>
<jvmArg>-Xss128m</jvmArg>
- <jvmArg>-Xms1024m</jvmArg>
- <jvmArg>-Xmx3200m</jvmArg>
+ <jvmArg>-Xms4g</jvmArg>
+ <jvmArg>-Xmx4g</jvmArg>
<jvmArg>-XX:MaxMetaspaceSize=2g</jvmArg>
<jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
</jvmArgs>
@@ -2662,7 +2662,7 @@
<include>**/*Suite.java</include>
</includes>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <argLine>-ea -Xmx3200m -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+ <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
<environmentVariables>
<!--
Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
@@ -2713,7 +2713,7 @@
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>SparkTestSuite.txt</filereports>
- <argLine>-ea -Xmx3200m -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+ <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
<stderr/>
<environmentVariables>
<!--
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org