You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/03/31 05:39:06 UTC
[incubator-celeborn] branch main updated: [CELEBORN-497]Fix and enable JDK 11 for CI (#1401)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 61416a828 [CELEBORN-497]Fix and enable JDK 11 for CI (#1401)
61416a828 is described below
commit 61416a828d59dee01e88b4901a3850896050144b
Author: Keyong Zhou <zh...@apache.org>
AuthorDate: Fri Mar 31 13:39:02 2023 +0800
[CELEBORN-497]Fix and enable JDK 11 for CI (#1401)
---
.github/workflows/maven.yml | 38 +++++++++++++++++-
.../network/server/memory/MemoryManager.java | 46 +++++++++++-----------
pom.xml | 3 +-
3 files changed, 62 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index a1510de46..1078a4276 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -21,6 +21,7 @@ jobs:
matrix:
java:
- 8
+ - 11
steps:
- uses: actions/checkout@v2
- name: Setup JDK ${{ matrix.java }}
@@ -43,7 +44,7 @@ jobs:
if: matrix.java == 8
uses: codecov/codecov-action@v3
- spark:
+ spark2:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -52,6 +53,40 @@ jobs:
- 8
spark:
- '2.4'
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test with Maven
+ run: |
+ SPARK_BINARY_VERSION=${{ matrix.spark }}
+ SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
+ PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
+ TEST_MODULES="client-spark/common,client-spark/spark-${SPARK_MAJOR_VERSION},client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
+ build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
+ build/mvn $PROFILES -pl $TEST_MODULES test
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v3
+ with:
+ name: spark-${{ matrix.spark }}-unit-test-log
+ path: |
+ **/target/unit-tests.log
+
+ spark3:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 8
+ - 11
+ spark:
- '3.0'
- '3.1'
- '3.2'
@@ -88,6 +123,7 @@ jobs:
matrix:
java:
- 8
+ - 11
flink:
- '1.14'
steps:
diff --git a/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java b/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java
index 88a73024a..be2b302ef 100644
--- a/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java
+++ b/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java
@@ -32,6 +32,8 @@ import java.util.concurrent.atomic.LongAdder;
import com.google.common.base.Preconditions;
import io.netty.buffer.ByteBuf;
import io.netty.util.internal.PlatformDependent;
+import org.apache.commons.lang3.JavaVersion;
+import org.apache.commons.lang3.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,29 +122,27 @@ public class MemoryManager {
double shuffleStorageRatio,
long checkInterval,
long reportInterval) {
- String[][] providers =
- new String[][] {
- {"sun.misc.VM", "maxDirectMemory"},
- {"jdk.internal.misc.VM", "maxDirectMemory"}
- };
-
- Method maxMemMethod = null;
- for (String[] provider : providers) {
- String clazz = provider[0];
- String method = provider[1];
- try {
- Class<?> vmClass = Class.forName(clazz);
- maxMemMethod = vmClass.getDeclaredMethod(method);
-
- maxMemMethod.setAccessible(true);
- maxDirectorMemory = (long) maxMemMethod.invoke(null);
- break;
- } catch (ClassNotFoundException
- | NoSuchMethodException
- | IllegalAccessException
- | InvocationTargetException ignored) {
- // Ignore Exception
- }
+ String[] provider;
+ if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_10)) {
+ provider = new String[] {"jdk.internal.misc.VM", "maxDirectMemory"};
+ } else {
+ provider = new String[] {"sun.misc.VM", "maxDirectMemory"};
+ }
+
+ Method maxMemMethod;
+ String clazz = provider[0];
+ String method = provider[1];
+ try {
+ Class<?> vmClass = Class.forName(clazz);
+ maxMemMethod = vmClass.getDeclaredMethod(method);
+
+ maxDirectorMemory = (long) maxMemMethod.invoke(null);
+ } catch (ClassNotFoundException
+ | NoSuchMethodException
+ | IllegalAccessException
+ | InvocationTargetException ignored) {
+ System.out.println("exception " + ignored);
+ // Ignore Exception
}
Preconditions.checkArgument(maxDirectorMemory > 0);
Preconditions.checkArgument(pauseReplicateRatio > pausePushDataRatio);
diff --git a/pom.xml b/pom.xml
index 9a109d5d9..71baf9abd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,8 @@
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
- --add-opens=java.base/sun.util.calendar=ALL-UNNAMED</extraJavaTestArgs>
+ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+ --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED</extraJavaTestArgs>
<mavenCentralId>central</mavenCentralId>
<mavenCentralName>Maven Central</mavenCentralName>