You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2025/05/10 12:35:41 UTC
(systemds) branch main updated: [MINOR] Fix robustness parameter server utils and tests
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 5f390b3cf2 [MINOR] Fix robustness parameter server utils and tests
5f390b3cf2 is described below
commit 5f390b3cf2775371e4f6ee39def7d3269cf05af3
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sat May 10 14:35:04 2025 +0200
[MINOR] Fix robustness parameter server utils and tests
---
src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java | 4 ----
src/main/java/org/apache/sysds/parser/DMLProgram.java | 6 ++++--
.../sysds/runtime/controlprogram/paramserv/dp/DCLocalScheme.java | 3 ++-
.../sysds/runtime/controlprogram/paramserv/dp/DRLocalScheme.java | 3 ++-
.../sysds/runtime/controlprogram/paramserv/dp/ORLocalScheme.java | 3 ++-
.../java/org/apache/sysds/test/functions/frame/FrameEqualTest.java | 2 +-
.../apache/sysds/test/functions/paramserv/ParamservSyntaxTest.java | 1 -
.../sysds/test/functions/paramserv/SparkDataPartitionerTest.java | 4 +++-
8 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java
index 8f3279a069..2119826697 100644
--- a/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java
@@ -20,8 +20,6 @@
package org.apache.sysds.hops.rewrite;
import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.sysds.api.DMLScript;
import org.apache.sysds.common.Opcodes;
import org.apache.sysds.common.Types.DataType;
@@ -90,7 +88,6 @@ import java.util.LinkedHashMap;
import java.util.List;
public class HopRewriteUtils {
- private static final Log LOG = LogFactory.getLog(HopRewriteUtils.class.getName());
public static boolean isValueTypeCast( OpOp1 op ) {
return op == OpOp1.CAST_AS_BOOLEAN
@@ -1712,7 +1709,6 @@ public class HopRewriteUtils {
// If the function keys are incorrect this exception is caught for robustness in error messages for users.
// Intensionally only catching the exception!
// For debugging if for some reason the error we encountered was something else we LOG the error.
- LOG.error(ex);
return false;
}
}
diff --git a/src/main/java/org/apache/sysds/parser/DMLProgram.java b/src/main/java/org/apache/sysds/parser/DMLProgram.java
index 13757cab56..2f69cb7ea0 100644
--- a/src/main/java/org/apache/sysds/parser/DMLProgram.java
+++ b/src/main/java/org/apache/sysds/parser/DMLProgram.java
@@ -89,12 +89,14 @@ public class DMLProgram
*/
public FunctionStatementBlock getFunctionStatementBlock(String fkey) {
String[] tmp = splitFunctionKey(fkey);
- return getFunctionStatementBlock(tmp[0], tmp[1]);
+ return tmp.length < 2 ? null :
+ getFunctionStatementBlock(tmp[0], tmp[1]);
}
public void removeFunctionStatementBlock(String fkey) {
String[] tmp = splitFunctionKey(fkey);
- removeFunctionStatementBlock(tmp[0], tmp[1]);
+ if( tmp.length == 2 )
+ removeFunctionStatementBlock(tmp[0], tmp[1]);
}
public FunctionStatementBlock getFunctionStatementBlock(String namespaceKey, String functionName) {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DCLocalScheme.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DCLocalScheme.java
index 5fbe02281b..f5cb19b96d 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DCLocalScheme.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DCLocalScheme.java
@@ -53,7 +53,8 @@ public class DCLocalScheme extends DataPartitionLocalScheme {
}
private static List<MatrixObject> doPartitioning(int k, MatrixBlock mb) {
- return partition(k, mb).stream().map(ParamservUtils::newMatrixObject).collect(Collectors.toList());
+ return partition(k, mb).stream()
+ .map(m -> ParamservUtils.newMatrixObject(m, false)).collect(Collectors.toList());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DRLocalScheme.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DRLocalScheme.java
index e17e41767b..a59b0cbc61 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DRLocalScheme.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/DRLocalScheme.java
@@ -49,7 +49,8 @@ public class DRLocalScheme extends DataPartitionLocalScheme {
}
private static List<MatrixObject> internalDoPartitioning(int k, MatrixBlock mb, MatrixBlock permutation) {
- return partition(k, mb, permutation).stream().map(ParamservUtils::newMatrixObject).collect(Collectors.toList());
+ return partition(k, mb, permutation).stream()
+ .map(m -> ParamservUtils.newMatrixObject(m, false)).collect(Collectors.toList());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/ORLocalScheme.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/ORLocalScheme.java
index 4cf693148e..17acde34a7 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/ORLocalScheme.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/dp/ORLocalScheme.java
@@ -46,7 +46,8 @@ public class ORLocalScheme extends DataPartitionLocalScheme {
}
private static List<MatrixObject> doPartitioning(int k, MatrixBlock mb, List<MatrixBlock> permutations) {
- return partition(k, mb, permutations).stream().map(ParamservUtils::newMatrixObject).collect(Collectors.toList());
+ return partition(k, mb, permutations).stream()
+ .map(m -> ParamservUtils.newMatrixObject(m, false)).collect(Collectors.toList());
}
@Override
diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameEqualTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameEqualTest.java
index fcd4321301..c6372d642b 100644
--- a/src/test/java/org/apache/sysds/test/functions/frame/FrameEqualTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameEqualTest.java
@@ -134,7 +134,7 @@ public class FrameEqualTest extends AutomatedTestBase {
Types.ExecMode platformOld = setExecMode(et);
boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
- setOutputBuffering(true);
+ //setOutputBuffering(true);
try {
getAndLoadTestConfiguration(TEST_NAME);
diff --git a/src/test/java/org/apache/sysds/test/functions/paramserv/ParamservSyntaxTest.java b/src/test/java/org/apache/sysds/test/functions/paramserv/ParamservSyntaxTest.java
index 7d4b5f4b25..5443fad94b 100644
--- a/src/test/java/org/apache/sysds/test/functions/paramserv/ParamservSyntaxTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/paramserv/ParamservSyntaxTest.java
@@ -98,6 +98,5 @@ public class ParamservSyntaxTest extends AutomatedTestBase {
programArgs = new String[] { "-explain" };
fullDMLScriptName = HOME + testname + ".dml";
runTest(true, exceptionExpected, exceptionClass, errmsg, -1);
- setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/paramserv/SparkDataPartitionerTest.java b/src/test/java/org/apache/sysds/test/functions/paramserv/SparkDataPartitionerTest.java
index 4dde72f103..2bf6a18922 100644
--- a/src/test/java/org/apache/sysds/test/functions/paramserv/SparkDataPartitionerTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/paramserv/SparkDataPartitionerTest.java
@@ -49,7 +49,9 @@ public class SparkDataPartitionerTest extends BaseDataPartitionerTest {
private Map<Integer, Tuple2<MatrixBlock, MatrixBlock>> doPartitioning(Statement.PSScheme scheme) {
MatrixBlock[] mbs = generateData();
- return SparkParamservUtils.doPartitionOnSpark(_sec, ParamservUtils.newMatrixObject(mbs[0]), ParamservUtils.newMatrixObject(mbs[1]), scheme, WORKER_NUM).collectAsMap();
+ return SparkParamservUtils.doPartitionOnSpark(_sec,
+ ParamservUtils.newMatrixObject(mbs[0], false),
+ ParamservUtils.newMatrixObject(mbs[1]), scheme, WORKER_NUM).collectAsMap();
}
@Test