You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2024/08/27 10:20:16 UTC
(systemds) branch main updated: [SYSTEMDS-3728] Extended memory checker util for windows OS and tests
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 64d1618f90 [SYSTEMDS-3728] Extended memory checker util for windows OS and tests
64d1618f90 is described below
commit 64d1618f901538f72a27b2437ba240748051d926
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Tue Aug 27 12:19:37 2024 +0200
[SYSTEMDS-3728] Extended memory checker util for windows OS and tests
---
.github/workflows/javaTests.yml | 2 +-
.../org/apache/sysds/utils/SettingsChecker.java | 35 +++++++++++++---
.../test/component/utils/SettingsCheckerTest.java | 46 ++++++++++++++++++++++
3 files changed, 76 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml
index df81841638..224d74bd4f 100644
--- a/.github/workflows/javaTests.yml
+++ b/.github/workflows/javaTests.yml
@@ -54,7 +54,7 @@ jobs:
"**.test.usertest.**",
"**.component.c**.**",
"**.component.e**.**,**.component.f**.**,**.component.m**.**",
- "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**",
+ "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**,**.component.u**.**",
"**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**",
"**.functions.blocks.**,**.functions.data.rand.**,",
"**.functions.countDistinct.**,**.functions.countDistinctApprox.**",
diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java
index 210263813b..f842134ab6 100644
--- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java
+++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java
@@ -43,7 +43,7 @@ public interface SettingsChecker {
}
}
- private static void checkMemorySetting() {
+ public static void checkMemorySetting() {
long JRE_Mem_Byte = Runtime.getRuntime().maxMemory();
long Sys_Mem_Byte = maxMemMachine() * 1024;
// Default 500MB
@@ -52,17 +52,17 @@ public interface SettingsChecker {
final long Logging_Limit = 1024L * 1024 * 1024 * 10;
if(JRE_Mem_Byte <= DefaultJava_500MB) {
- String st = byteMemoryToHumanReadableString(JRE_Mem_Byte);
+ String st = byteMemoryToString(JRE_Mem_Byte);
LOG.warn("Low memory budget set of: " + st + " this should most likely be increased");
}
else if(JRE_Mem_Byte < Logging_Limit && JRE_Mem_Byte * 10 < Sys_Mem_Byte) {
- String st = byteMemoryToHumanReadableString(JRE_Mem_Byte);
- String sm = byteMemoryToHumanReadableString(Sys_Mem_Byte);
+ String st = byteMemoryToString(JRE_Mem_Byte);
+ String sm = byteMemoryToString(Sys_Mem_Byte);
LOG.warn("Low memory budget of total: " + sm + " set to: " + st);
}
}
- private static long maxMemMachine() {
+ public static long maxMemMachine() {
String sys = System.getProperty("os.name");
if("Linux".equals(sys)) {
return maxMemMachineLinux();
@@ -70,6 +70,9 @@ public interface SettingsChecker {
else if(sys.contains("Mac OS")) {
return maxMemMachineOSX();
}
+ else if(sys.startsWith("Windows")) {
+ return maxMemMachineWin();
+ }
else {
return -1;
}
@@ -100,6 +103,26 @@ public interface SettingsChecker {
throw new RuntimeException(e);
}
}
+
+ private static long maxMemMachineWin() {
+ try {
+ String command = "wmic memorychip get capacity";
+ Runtime rt = Runtime.getRuntime();
+ Process pr = rt.exec(command);
+ String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n");
+ //skip header, and aggregate DIMM capacities
+ long capacity = 0;
+ for( int i=1; i<memStr.length; i++ ) {
+ String tmp = memStr[i].trim();
+ if( tmp.length() > 0 )
+ capacity += Long.parseLong(tmp);
+ }
+ return capacity;
+ }
+ catch(IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
/**
* Converts a number of bytes in a long to a human readable string with GB, MB, KB and B.
@@ -107,7 +130,7 @@ public interface SettingsChecker {
* @param bytes Number of bytes.
* @return A human readable string
*/
- public static String byteMemoryToHumanReadableString(long bytes) {
+ public static String byteMemoryToString(long bytes) {
if(bytes > 1000000000)
return String.format("%6d GB", bytes / 1024 / 1024 / 1024);
else if(bytes > 1000000)
diff --git a/src/test/java/org/apache/sysds/test/component/utils/SettingsCheckerTest.java b/src/test/java/org/apache/sysds/test/component/utils/SettingsCheckerTest.java
new file mode 100644
index 0000000000..1d2d297e4c
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/component/utils/SettingsCheckerTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.utils;
+
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.utils.SettingsChecker;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SettingsCheckerTest extends AutomatedTestBase {
+
+ @Override
+ public void setUp() {}
+
+ @Test
+ public void testCheckMemory() {
+ try {
+ SettingsChecker.check();
+ SettingsChecker.checkMemorySetting();
+ long mem = SettingsChecker.maxMemMachine();
+ String memStr = SettingsChecker.byteMemoryToString(mem);
+ System.out.println(memStr);
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ Assert.fail();
+ }
+ }
+}