You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2020/10/31 22:07:12 UTC
[systemds] 04/05: [MINOR] Fix null-pointer exceptions in new
federated I/O tests
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit e96e1cbafc53e90f64c08899cccbc1f02a9e46b3
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sat Oct 31 21:41:51 2020 +0100
[MINOR] Fix null-pointer exceptions in new federated I/O tests
In order to allow running these tests in isolation they need to enable
output buffering and cannot rely on running in the same JVM where some
other test might have already enabled it.
---
.../federated/io/FederatedReaderTest.java | 31 +++++++---------------
.../federated/io/FederatedWriterTest.java | 26 +++++++-----------
2 files changed, 19 insertions(+), 38 deletions(-)
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java
index c14ac1d..8fa2063 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java
@@ -21,7 +21,6 @@ package org.apache.sysds.test.functions.federated.io;
import java.util.Arrays;
import java.util.Collection;
-import org.apache.sysds.api.DMLScript;
import org.apache.sysds.common.Types;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
@@ -70,14 +69,10 @@ public class FederatedReaderTest extends AutomatedTestBase {
}
public void federatedRead(Types.ExecMode execMode) {
- boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
- Types.ExecMode platformOld = rtplatform;
- rtplatform = execMode;
- if(rtplatform == Types.ExecMode.SPARK) {
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
- }
+ Types.ExecMode oldPlatform = setExecMode(execMode);
getAndLoadTestConfiguration(TEST_NAME);
-
+ setOutputBuffering(true);
+
// write input matrices
int halfRows = rows / 2;
long[][] begins = new long[][] {new long[] {0, 0}, new long[] {halfRows, 0}};
@@ -95,15 +90,9 @@ public class FederatedReaderTest extends AutomatedTestBase {
Thread t2 = startLocalFedWorkerThread(port2);
String host = "localhost";
- MatrixObject fed = FederatedTestObjectConstructor.constructFederatedInput(rows,
- cols,
- blocksize,
- host,
- begins,
- ends,
- new int[] {port1, port2},
- new String[] {input("X1"), input("X2")},
- input("X.json"));
+ MatrixObject fed = FederatedTestObjectConstructor.constructFederatedInput(
+ rows, cols, blocksize, host, begins, ends, new int[] {port1, port2},
+ new String[] {input("X1"), input("X2")}, input("X.json"));
writeInputFederatedWithMTD("X.json", fed, null);
try {
@@ -120,16 +109,16 @@ public class FederatedReaderTest extends AutomatedTestBase {
Assert.assertTrue(heavyHittersContainsString("fed_uak+"));
// Verify output
Assert.assertEquals(Double.parseDouble(refOut.split("\n")[0]),
- Double.parseDouble(out.split("\n")[0]),
- 0.00001);
+ Double.parseDouble(out.split("\n")[0]), 0.00001);
}
catch(Exception e) {
e.printStackTrace();
Assert.assertTrue(false);
}
+ finally {
+ resetExecMode(oldPlatform);
+ }
TestUtils.shutdownThreads(t1, t2);
- rtplatform = platformOld;
- DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java
index e03474d..587da82 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java
@@ -21,8 +21,7 @@ package org.apache.sysds.test.functions.federated.io;
import java.util.Arrays;
import java.util.Collection;
-import org.apache.sysds.api.DMLScript;
-import org.apache.sysds.common.Types;
+import org.apache.sysds.common.Types.ExecMode;
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
@@ -65,17 +64,13 @@ public class FederatedWriterTest extends AutomatedTestBase {
@Test
public void federatedSinglenodeWrite() {
- federatedWrite(Types.ExecMode.SINGLE_NODE);
+ federatedWrite(ExecMode.SINGLE_NODE);
}
- public void federatedWrite(Types.ExecMode execMode) {
- boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
- Types.ExecMode platformOld = rtplatform;
- rtplatform = execMode;
- if(rtplatform == Types.ExecMode.SPARK) {
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
- }
+ public void federatedWrite(ExecMode execMode) {
+ ExecMode oldPlatform = setExecMode(execMode);
getAndLoadTestConfiguration(TEST_NAME);
+ setOutputBuffering(true);
// write input matrices
int halfRows = rows / 2;
@@ -97,10 +92,7 @@ public class FederatedWriterTest extends AutomatedTestBase {
fullDMLScriptName = SCRIPT_DIR + "functions/federated/io/FederatedReaderTestCreate.dml";
programArgs = new String[] {"-stats", "-explain", "-args", input("X1"), input("X2"), port1 + "", port2 + "",
input("X.json")};
- // String writer = runTest(null).toString();
runTest(null);
- // LOG.error(writer);
- // LOG.error("Writing Done");
// Run reference dml script with normal matrix
fullDMLScriptName = SCRIPT_DIR + "functions/federated/io/FederatedReaderTest.dml";
@@ -120,16 +112,16 @@ public class FederatedWriterTest extends AutomatedTestBase {
// Verify output
Assert.assertEquals(Double.parseDouble(refOut.split("\n")[0]),
- Double.parseDouble(out.split("\n")[0]),
- 0.00001);
+ Double.parseDouble(out.split("\n")[0]), 0.00001);
}
catch(Exception e) {
e.printStackTrace();
Assert.assertTrue(false);
}
+ finally {
+ resetExecMode(oldPlatform);
+ }
TestUtils.shutdownThreads(t1, t2);
- rtplatform = platformOld;
- DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
}
}