You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2020/07/24 08:57:02 UTC
[systemds] branch master updated: [SYSTEMDS-2578] Small
improvements to fed tests
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new b1a5247 [SYSTEMDS-2578] Small improvements to fed tests
b1a5247 is described below
commit b1a5247a82d5650c1365f49fd1305ffe974de638
Author: Kevin Innerebner <ke...@yahoo.com>
AuthorDate: Tue Jul 21 17:59:53 2020 +0200
[SYSTEMDS-2578] Small improvements to fed tests
Make use of TestUtils.federatedAddress and prefer nvargs
over args for DML execution. Additionally some minor reformatting.
Closes #998
---
src/test/java/org/apache/sysds/test/TestUtils.java | 4 +
.../federated/FederatedConstructionTest.java | 58 +++++------
.../functions/federated/FederatedL2SVMTest.java | 21 ++--
.../FederatedMatrixScalarOperationsTest.java | 115 ++++++++++-----------
.../functions/federated/FederatedMultiplyTest.java | 23 ++---
.../functions/federated/FederatedRCBindTest.java | 19 ++--
.../test/functions/federated/FederatedSumTest.java | 23 ++---
.../privacy/FederatedWorkerHandlerTest.java | 19 ++--
.../federated/FederatedFrameConstructionTest.dml | 5 +-
.../functions/federated/FederatedL2SVMTest.dml | 8 +-
.../federated/FederatedMatrixConstructionTest.dml | 6 +-
.../functions/federated/FederatedMultiplyTest.dml | 4 +-
.../functions/federated/FederatedRCBindTest.dml | 8 +-
.../functions/federated/FederatedSumTest.dml | 8 +-
.../FederatedMatrixAdditionScalar.dml | 6 +-
.../FederatedMatrixMultiplicationScalar.dml | 6 +-
.../FederatedMatrixSubtractionScalar.dml | 6 +-
...atrix.dml => FederatedScalarAdditionMatrix.dml} | 6 +-
...dml => FederatedScalarMultiplicationMatrix.dml} | 6 +-
...ix.dml => FederatedScalarSubtractionMatrix.dml} | 6 +-
20 files changed, 168 insertions(+), 189 deletions(-)
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java
index 8819f56..1078658 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -2459,6 +2459,10 @@ public class TestUtils
}
}
+ public static String federatedAddress(int port, String input) {
+ return federatedAddress("localhost", port, input);
+ }
+
public static String federatedAddress(String host, int port, String input) {
return host + ':' + port + '/' + input;
}
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/FederatedConstructionTest.java b/src/test/java/org/apache/sysds/test/functions/federated/FederatedConstructionTest.java
index a12f1a8..340234b 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/FederatedConstructionTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/FederatedConstructionTest.java
@@ -40,27 +40,23 @@ import java.util.List;
@RunWith(value = Parameterized.class)
@net.jcip.annotations.NotThreadSafe
public class FederatedConstructionTest extends AutomatedTestBase {
-
+
private final static String TEST_DIR = "functions/federated/";
private final static String TEST_NAME = "FederatedConstructionTest";
private final static String TEST_CLASS_DIR = TEST_DIR + FederatedConstructionTest.class.getSimpleName() + "/";
public static final String MATRIX_TEST_FILE_NAME = "FederatedMatrixConstructionTest";
public static final String FRAME_TEST_FILE_NAME = "FederatedFrameConstructionTest";
-
- private int blocksize = 1024;
- private int rows, cols;
-
- public FederatedConstructionTest(int rows, int cols) {
- this.rows = rows;
- this.cols = cols;
- }
+ private static final int blocksize = 1024;
+ @Parameterized.Parameter()
+ public int rows;
+ @Parameterized.Parameter(1)
+ public int cols;
@Parameterized.Parameters
public static Collection<Object[]> data() {
// cols have to be dividable by 4 for Frame tests
- Object[][] data = new Object[][] {{1, 1024}, {8, 256}, {256, 8}, {1024, 4}, {16, 2048}, {2048, 32}};
- return Arrays.asList(data);
+ return Arrays.asList(new Object[][] {{1, 1024}, {8, 256}, {256, 8}, {1024, 4}, {16, 2048}, {2048, 32}});
}
@Override
@@ -86,36 +82,38 @@ public class FederatedConstructionTest extends AutomatedTestBase {
writeInputMatrixWithMTD("A", A, false, new MatrixCharacteristics(rows, cols, blocksize, rows * cols));
federatedConstruction(execMode, MATRIX_TEST_FILE_NAME, "A", null);
}
-
+
@Test
public void federatedFrameConstructionCP() throws IOException {
federatedFrameConstruction(Types.ExecMode.SINGLE_NODE);
}
-
- /* like other federated functionality, SPARK execution mode is not yet working (waiting for better integration
- of federated instruction building, like propagating information that object is federated)
- @Test
- public void federatedFrameConstructionSP() throws IOException {
- federatedFrameConstruction(Types.ExecMode.SPARK);
- }*/
-
+
+ /*
+ * like other federated functionality, SPARK execution mode is not yet working (waiting for better integration of
+ * federated instruction building, like propagating information that object is federated)
+ *
+ * @Test public void federatedFrameConstructionSP() throws IOException {
+ * federatedFrameConstruction(Types.ExecMode.SPARK); }
+ */
+
public void federatedFrameConstruction(Types.ExecMode execMode) throws IOException {
getAndLoadTestConfiguration(TEST_NAME);
// write input matrix
double[][] A = getRandomMatrix(rows, cols, -1, 1, 1, 1234);
-
- List<Types.ValueType> schemaList = new ArrayList<>(Collections.nCopies(cols/4, Types.ValueType.STRING));
- schemaList.addAll(Collections.nCopies(cols/4, Types.ValueType.FP64));
- schemaList.addAll(Collections.nCopies(cols/4, Types.ValueType.INT64));
- schemaList.addAll(Collections.nCopies(cols/4, Types.ValueType.BOOLEAN));
-
+
+ List<Types.ValueType> schemaList = new ArrayList<>(Collections.nCopies(cols / 4, Types.ValueType.STRING));
+ schemaList.addAll(Collections.nCopies(cols / 4, Types.ValueType.FP64));
+ schemaList.addAll(Collections.nCopies(cols / 4, Types.ValueType.INT64));
+ schemaList.addAll(Collections.nCopies(cols / 4, Types.ValueType.BOOLEAN));
+
Types.ValueType[] schema = new Types.ValueType[cols];
schemaList.toArray(schema);
writeInputFrameWithMTD("A", A, false, schema, FileFormat.BINARY);
federatedConstruction(execMode, FRAME_TEST_FILE_NAME, "A", schema);
}
- public void federatedConstruction(Types.ExecMode execMode, String testFile, String inputIdentifier, Types.ValueType[] schema) {
+ public void federatedConstruction(Types.ExecMode execMode, String testFile, String inputIdentifier,
+ Types.ValueType[] schema) {
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
Types.ExecMode platformOld = rtplatform;
@@ -142,12 +140,12 @@ public class FederatedConstructionTest extends AutomatedTestBase {
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
}
fullDMLScriptName = HOME + testFile + ".dml";
- programArgs = new String[] {"-args", "\"localhost:" + port + "/" + input(inputIdentifier) + "\"",
- Integer.toString(rows), Integer.toString(cols), Integer.toString(rows * 2), output("B")};
+ programArgs = new String[] {"-nvargs", "in=" + TestUtils.federatedAddress(port, input(inputIdentifier)),
+ "rows=" + rows, "cols=" + cols, "out=" + output("B")};
runTest(true, false, null, -1);
// compare via files
- if (schema != null)
+ if(schema != null)
compareResults(schema);
else
compareResults(1e-12);
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/FederatedL2SVMTest.java b/src/test/java/org/apache/sysds/test/functions/federated/FederatedL2SVMTest.java
index d614ec6..e55cfc9 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/FederatedL2SVMTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/FederatedL2SVMTest.java
@@ -41,12 +41,10 @@ public class FederatedL2SVMTest extends AutomatedTestBase {
private final static String TEST_CLASS_DIR = TEST_DIR + FederatedL2SVMTest.class.getSimpleName() + "/";
private final static int blocksize = 1024;
- private int rows, cols;
-
- public FederatedL2SVMTest(int rows, int cols) {
- this.rows = rows;
- this.cols = cols;
- }
+ @Parameterized.Parameter()
+ public int rows;
+ @Parameterized.Parameter(1)
+ public int cols;
@Override
public void setUp() {
@@ -57,8 +55,7 @@ public class FederatedL2SVMTest extends AutomatedTestBase {
@Parameterized.Parameters
public static Collection<Object[]> data() {
// rows have to be even and > 1
- Object[][] data = new Object[][] {{2, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}};
- return Arrays.asList(data);
+ return Arrays.asList(new Object[][] {{2, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}});
}
@Test
@@ -79,7 +76,7 @@ public class FederatedL2SVMTest extends AutomatedTestBase {
if(rtplatform == Types.ExecMode.SPARK) {
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
}
- Thread t1 = null, t2 = null;
+ Thread t1, t2;
getAndLoadTestConfiguration(TEST_NAME);
String HOME = SCRIPT_DIR + TEST_DIR;
@@ -114,9 +111,9 @@ public class FederatedL2SVMTest extends AutomatedTestBase {
// Run actual dml script with federated matrix
fullDMLScriptName = HOME + TEST_NAME + ".dml";
- programArgs = new String[] {"-args", "\"localhost:" + port1 + "/" + input("X1") + "\"",
- "\"localhost:" + port2 + "/" + input("X2") + "\"", Integer.toString(rows), Integer.toString(cols),
- Integer.toString(halfRows), input("Y"), output("Z")};
+ programArgs = new String[] {"-nvargs", "in_X1=" + TestUtils.federatedAddress(port1, input("X1")),
+ "in_X2=" + TestUtils.federatedAddress(port2, input("X2")), "rows=" + rows, "cols=" + cols,
+ "in_Y=" + input("Y"), "out=" + output("Z")};
runTest(true, false, null, -1);
// compare via files
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/FederatedMatrixScalarOperationsTest.java b/src/test/java/org/apache/sysds/test/functions/federated/FederatedMatrixScalarOperationsTest.java
index c347ed8..f67bd9b 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/FederatedMatrixScalarOperationsTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/FederatedMatrixScalarOperationsTest.java
@@ -19,7 +19,6 @@
package org.apache.sysds.test.functions.federated;
-
import org.junit.Test;
import org.junit.runners.Parameterized;
import org.junit.runner.RunWith;
@@ -33,66 +32,67 @@ import java.util.Arrays;
import static java.lang.Thread.sleep;
-
@RunWith(Parameterized.class)
@net.jcip.annotations.NotThreadSafe
-public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase
-{
+public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase {
@Parameterized.Parameters
public static Iterable<Object[]> data() {
- return Arrays.asList(new Object[][] {
- { 100, 100 },
- { 10000, 100 },
- });
+ return Arrays.asList(new Object[][] {{100, 100}, {10000, 100},});
}
- //internals 4 parameterized tests
- @Parameterized.Parameter(0)
- public Integer rows;
+ // internals 4 parameterized tests
+ @Parameterized.Parameter()
+ public int rows;
@Parameterized.Parameter(1)
- public Integer cols;
+ public int cols;
- //System test paths
+ // System test paths
private static final String TEST_DIR = "functions/federated/matrix_scalar/";
- private static final String TEST_CLASS_DIR = TEST_DIR + FederatedMatrixScalarOperationsTest.class.getSimpleName() + "/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + FederatedMatrixScalarOperationsTest.class.getSimpleName()
+ + "/";
private static final String TEST_PROG_MATRIX_ADDITION_SCALAR = "FederatedMatrixAdditionScalar";
private static final String TEST_PROG_MATRIX_SUBTRACTION_SCALAR = "FederatedMatrixSubtractionScalar";
private static final String TEST_PROG_MATRIX_MULTIPLICATION_SCALAR = "FederatedMatrixMultiplicationScalar";
- private static final String TEST_PROG_SCALAR_ADDITION_MATRIX = "ScalarAdditionFederatedMatrix";
- private static final String TEST_PROG_SCALAR_SUBTRACTION_MATRIX = "ScalarSubtractionFederatedMatrix";
- private static final String TEST_PROG_SCALAR_MULTIPLICATION_MATRIX = "ScalarMultiplicationFederatedMatrix";
+ private static final String TEST_PROG_SCALAR_ADDITION_MATRIX = "FederatedScalarAdditionMatrix";
+ private static final String TEST_PROG_SCALAR_SUBTRACTION_MATRIX = "FederatedScalarSubtractionMatrix";
+ private static final String TEST_PROG_SCALAR_MULTIPLICATION_MATRIX = "FederatedScalarMultiplicationMatrix";
private static final String FEDERATED_WORKER_HOST = "localhost";
private static final int FEDERATED_WORKER_PORT = 1222;
@Override
- public void setUp()
- {
- //Save Result to File R
- addTestConfiguration(new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_MATRIX_ADDITION_SCALAR, new String [] {"R"}));
- addTestConfiguration(new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_MATRIX_SUBTRACTION_SCALAR, new String [] {"R"}));
- addTestConfiguration(new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_MATRIX_MULTIPLICATION_SCALAR, new String [] {"R"}));
- addTestConfiguration(new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_SCALAR_ADDITION_MATRIX, new String [] {"R"}));
- addTestConfiguration(new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_SCALAR_SUBTRACTION_MATRIX, new String [] {"R"}));
- addTestConfiguration(new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_SCALAR_MULTIPLICATION_MATRIX, new String [] {"R"}));
+ public void setUp() {
+ // Save Result to File R
+ addTestConfiguration(
+ new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_MATRIX_ADDITION_SCALAR, new String[] {"R"}));
+ addTestConfiguration(
+ new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_MATRIX_SUBTRACTION_SCALAR, new String[] {"R"}));
+ addTestConfiguration(
+ new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_MATRIX_MULTIPLICATION_SCALAR, new String[] {"R"}));
+ addTestConfiguration(
+ new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_SCALAR_ADDITION_MATRIX, new String[] {"R"}));
+ addTestConfiguration(
+ new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_SCALAR_SUBTRACTION_MATRIX, new String[] {"R"}));
+ addTestConfiguration(
+ new TestConfiguration(TEST_CLASS_DIR, TEST_PROG_SCALAR_MULTIPLICATION_MATRIX, new String[] {"R"}));
}
-
- @Test
+
+ @Test
public void testFederatedMatrixAdditionScalar() {
getAndLoadTestConfiguration(TEST_PROG_MATRIX_ADDITION_SCALAR);
double[][] m = getRandomMatrix(this.rows, this.cols, -1, 1, 1.0, 1);
writeInputMatrixWithMTD("M", m, true);
- int s = TestUtils.getRandomInt();
+ int scalar = TestUtils.getRandomInt();
double[][] r = new double[rows][cols];
for(int i = 0; i < rows; i++) {
for(int j = 0; j < cols; j++) {
- r[i][j] = m[i][j] + s;
+ r[i][j] = m[i][j] + scalar;
}
}
writeExpectedMatrix("R", r);
- runGenericTest(TEST_PROG_MATRIX_ADDITION_SCALAR, s);
+ runGenericTest(TEST_PROG_MATRIX_ADDITION_SCALAR, scalar);
}
@Test
@@ -101,16 +101,16 @@ public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase
double[][] m = getRandomMatrix(this.rows, this.cols, -1, 1, 1.0, 1);
writeInputMatrixWithMTD("M", m, true);
- int s = TestUtils.getRandomInt();
+ int scalar = TestUtils.getRandomInt();
double[][] r = new double[rows][cols];
for(int i = 0; i < rows; i++) {
for(int j = 0; j < cols; j++) {
- r[i][j] = m[i][j] - s;
+ r[i][j] = m[i][j] - scalar;
}
}
writeExpectedMatrix("R", r);
- runGenericTest(TEST_PROG_MATRIX_SUBTRACTION_SCALAR, s);
+ runGenericTest(TEST_PROG_MATRIX_SUBTRACTION_SCALAR, scalar);
}
@Test
@@ -119,16 +119,16 @@ public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase
double[][] m = getRandomMatrix(this.rows, this.cols, -1, 1, 1.0, 1);
writeInputMatrixWithMTD("M", m, true);
- int s = TestUtils.getRandomInt();
+ int scalar = TestUtils.getRandomInt();
double[][] r = new double[rows][cols];
for(int i = 0; i < rows; i++) {
for(int j = 0; j < cols; j++) {
- r[i][j] = m[i][j] * s;
+ r[i][j] = m[i][j] * scalar;
}
}
writeExpectedMatrix("R", r);
- runGenericTest(TEST_PROG_MATRIX_MULTIPLICATION_SCALAR, s);
+ runGenericTest(TEST_PROG_MATRIX_MULTIPLICATION_SCALAR, scalar);
}
@Test
@@ -137,16 +137,16 @@ public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase
double[][] m = getRandomMatrix(this.rows, this.cols, -1, 1, 1.0, 1);
writeInputMatrixWithMTD("M", m, true);
- int s = TestUtils.getRandomInt();
+ int scalar = TestUtils.getRandomInt();
double[][] r = new double[rows][cols];
for(int i = 0; i < rows; i++) {
for(int j = 0; j < cols; j++) {
- r[i][j] = m[i][j] + s;
+ r[i][j] = m[i][j] + scalar;
}
}
writeExpectedMatrix("R", r);
- runGenericTest(TEST_PROG_SCALAR_ADDITION_MATRIX, s);
+ runGenericTest(TEST_PROG_SCALAR_ADDITION_MATRIX, scalar);
}
@Test
@@ -155,16 +155,16 @@ public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase
double[][] m = getRandomMatrix(this.rows, this.cols, -1, 1, 1.0, 1);
writeInputMatrixWithMTD("M", m, true);
- int s = TestUtils.getRandomInt();
+ int scalar = TestUtils.getRandomInt();
double[][] r = new double[rows][cols];
for(int i = 0; i < rows; i++) {
for(int j = 0; j < cols; j++) {
- r[i][j] = s - m[i][j];
+ r[i][j] = scalar - m[i][j];
}
}
writeExpectedMatrix("R", r);
- runGenericTest(TEST_PROG_SCALAR_SUBTRACTION_MATRIX, s);
+ runGenericTest(TEST_PROG_SCALAR_SUBTRACTION_MATRIX, scalar);
}
@Test
@@ -173,23 +173,19 @@ public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase
double[][] m = getRandomMatrix(this.rows, this.cols, -1, 1, 1.0, 1);
writeInputMatrixWithMTD("M", m, true);
- int s = TestUtils.getRandomInt();
+ int scalar = TestUtils.getRandomInt();
double[][] r = new double[rows][cols];
for(int i = 0; i < rows; i++) {
for(int j = 0; j < cols; j++) {
- r[i][j] = m[i][j] * s;
+ r[i][j] = m[i][j] * scalar;
}
}
writeExpectedMatrix("R", r);
- runGenericTest(TEST_PROG_SCALAR_MULTIPLICATION_MATRIX, s);
+ runGenericTest(TEST_PROG_SCALAR_MULTIPLICATION_MATRIX, scalar);
}
-
-
-
- private void runGenericTest(String dmlFile, int s)
- {
+ private void runGenericTest(String dmlFile, int scalar) {
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
Types.ExecMode platformOld = rtplatform;
@@ -197,26 +193,23 @@ public class FederatedMatrixScalarOperationsTest extends AutomatedTestBase
try {
// we need the reference file to not be written to hdfs, so we get the correct format
rtplatform = Types.ExecMode.SINGLE_NODE;
- if (rtplatform == Types.ExecMode.SPARK) {
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
- }
programArgs = new String[] {"-w", Integer.toString(FEDERATED_WORKER_PORT)};
t = new Thread(() -> runTest(true, false, null, -1));
t.start();
sleep(FED_WORKER_WAIT);
fullDMLScriptName = SCRIPT_DIR + TEST_DIR + dmlFile + ".dml";
- programArgs = new String[]{"-args",
- TestUtils.federatedAddress(FEDERATED_WORKER_HOST, FEDERATED_WORKER_PORT, input("M")),
- Integer.toString(rows), Integer.toString(cols),
- Integer.toString(s),
- output("R")};
+ programArgs = new String[] {"-nvargs",
+ "in=" + TestUtils.federatedAddress(FEDERATED_WORKER_HOST, FEDERATED_WORKER_PORT, input("M")),
+ "rows=" + rows, "cols=" + cols, "scalar=" + scalar, "out=" + output("R")};
runTest(true, false, null, -1);
compareResults();
- } catch (InterruptedException e) {
+ }
+ catch(InterruptedException e) {
e.printStackTrace();
assert (false);
- } finally {
+ }
+ finally {
rtplatform = platformOld;
TestUtils.shutdownThread(t);
rtplatform = platformOld;
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/FederatedMultiplyTest.java b/src/test/java/org/apache/sysds/test/functions/federated/FederatedMultiplyTest.java
index 04f1970..67a74be 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/FederatedMultiplyTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/FederatedMultiplyTest.java
@@ -41,12 +41,10 @@ public class FederatedMultiplyTest extends AutomatedTestBase {
private final static String TEST_CLASS_DIR = TEST_DIR + FederatedMultiplyTest.class.getSimpleName() + "/";
private final static int blocksize = 1024;
- private int rows, cols;
-
- public FederatedMultiplyTest(int rows, int cols) {
- this.rows = rows;
- this.cols = cols;
- }
+ @Parameterized.Parameter()
+ public int rows;
+ @Parameterized.Parameter(1)
+ public int cols;
@Override
public void setUp() {
@@ -57,8 +55,7 @@ public class FederatedMultiplyTest extends AutomatedTestBase {
@Parameterized.Parameters
public static Collection<Object[]> data() {
// rows have to be even and > 1
- Object[][] data = new Object[][] {{2, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}};
- return Arrays.asList(data);
+ return Arrays.asList(new Object[][] {{2, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}});
}
@Test
@@ -115,12 +112,10 @@ public class FederatedMultiplyTest extends AutomatedTestBase {
// Run actual dml script with federated matrix
fullDMLScriptName = HOME + TEST_NAME + ".dml";
- programArgs = new String[] {"-nvargs",
- "X1=" + TestUtils.federatedAddress("localhost", port1, input("X1")),
- "X2=" + TestUtils.federatedAddress("localhost", port2, input("X2")),
- "Y1=" + TestUtils.federatedAddress("localhost", port1, input("Y1")),
- "Y2=" + TestUtils.federatedAddress("localhost", port2, input("Y2")), "r=" + rows, "c=" + cols,
- "hr=" + halfRows, "Z=" + output("Z")};
+ programArgs = new String[] {"-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")),
+ "X2=" + TestUtils.federatedAddress(port2, input("X2")),
+ "Y1=" + TestUtils.federatedAddress(port1, input("Y1")),
+ "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")};
runTest(true, false, null, -1);
// compare via files
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/FederatedRCBindTest.java b/src/test/java/org/apache/sysds/test/functions/federated/FederatedRCBindTest.java
index 5942816..93e1feb 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/FederatedRCBindTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/FederatedRCBindTest.java
@@ -41,17 +41,14 @@ public class FederatedRCBindTest extends AutomatedTestBase {
private final static String TEST_CLASS_DIR = TEST_DIR + FederatedRCBindTest.class.getSimpleName() + "/";
private final static int blocksize = 1024;
- private int rows, cols;
-
- public FederatedRCBindTest(int rows, int cols) {
- this.rows = rows;
- this.cols = cols;
- }
+ @Parameterized.Parameter()
+ public int rows;
+ @Parameterized.Parameter(1)
+ public int cols;
@Parameterized.Parameters
public static Collection<Object[]> data() {
- Object[][] data = new Object[][] {{1, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}};
- return Arrays.asList(data);
+ return Arrays.asList(new Object[][] {{1, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}});
}
@Override
@@ -74,7 +71,7 @@ public class FederatedRCBindTest extends AutomatedTestBase {
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
Types.ExecMode platformOld = rtplatform;
- Thread t = null;
+ Thread t;
getAndLoadTestConfiguration(TEST_NAME);
String HOME = SCRIPT_DIR + TEST_DIR;
@@ -100,8 +97,8 @@ public class FederatedRCBindTest extends AutomatedTestBase {
TestConfiguration config = availableTestConfigurations.get(TEST_NAME);
loadTestConfiguration(config);
fullDMLScriptName = HOME + TEST_NAME + ".dml";
- programArgs = new String[] {"-args", "\"localhost:" + port + "/" + input("A") + "\"", Integer.toString(rows),
- Integer.toString(cols), output("R"), output("C")};
+ programArgs = new String[] {"-nvargs", "in=" + TestUtils.federatedAddress(port, input("A")), "rows=" + rows,
+ "cols=" + cols, "out_R=" + output("R"), "out_C=" + output("C")};
runTest(true, false, null, -1);
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/FederatedSumTest.java b/src/test/java/org/apache/sysds/test/functions/federated/FederatedSumTest.java
index dd207dd..3118f01 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/FederatedSumTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/FederatedSumTest.java
@@ -41,17 +41,14 @@ public class FederatedSumTest extends AutomatedTestBase {
private final static String TEST_CLASS_DIR = TEST_DIR + FederatedSumTest.class.getSimpleName() + "/";
private final static int blocksize = 1024;
- private int rows, cols;
-
- public FederatedSumTest(int rows, int cols) {
- this.rows = rows;
- this.cols = cols;
- }
+ @Parameterized.Parameter()
+ public int rows;
+ @Parameterized.Parameter(1)
+ public int cols;
@Parameterized.Parameters
public static Collection<Object[]> data() {
- Object[][] data = new Object[][] {{1, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}};
- return Arrays.asList(data);
+ return Arrays.asList(new Object[][] {{2, 1000}, {10, 100}, {100, 10}, {1000, 1}, {10, 2000}, {2000, 10}});
}
@Override
@@ -75,13 +72,13 @@ public class FederatedSumTest extends AutomatedTestBase {
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
Types.ExecMode platformOld = rtplatform;
- Thread t = null;
+ Thread t;
getAndLoadTestConfiguration(TEST_NAME);
String HOME = SCRIPT_DIR + TEST_DIR;
- double[][] A = getRandomMatrix(rows, cols, -10, 10, 1, 1);
- writeInputMatrixWithMTD("A", A, false, new MatrixCharacteristics(rows, cols, blocksize, rows * cols));
+ double[][] A = getRandomMatrix(rows / 2, cols, -10, 10, 1, 1);
+ writeInputMatrixWithMTD("A", A, false, new MatrixCharacteristics(rows / 2, cols, blocksize, (rows / 2) * cols));
int port = getRandomAvailablePort();
t = startLocalFedWorker(port);
@@ -108,8 +105,8 @@ public class FederatedSumTest extends AutomatedTestBase {
TestConfiguration config = availableTestConfigurations.get(TEST_NAME);
loadTestConfiguration(config);
fullDMLScriptName = HOME + TEST_NAME + ".dml";
- programArgs = new String[] {"-args", "\"localhost:" + port + "/" + input("A") + "\"", Integer.toString(rows),
- Integer.toString(cols), Integer.toString(rows * 2), output("S"), output("R"), output("C")};
+ programArgs = new String[] {"-nvargs", "in=" + TestUtils.federatedAddress(port, input("A")), "rows=" + rows,
+ "cols=" + cols, "out_S=" + output("S"), "out_R=" + output("R"), "out_C=" + output("C")};
runTest(true, false, null, -1);
diff --git a/src/test/java/org/apache/sysds/test/functions/privacy/FederatedWorkerHandlerTest.java b/src/test/java/org/apache/sysds/test/functions/privacy/FederatedWorkerHandlerTest.java
index 8698448..19c45a0 100644
--- a/src/test/java/org/apache/sysds/test/functions/privacy/FederatedWorkerHandlerTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/privacy/FederatedWorkerHandlerTest.java
@@ -48,8 +48,8 @@ public class FederatedWorkerHandlerTest extends AutomatedTestBase {
private static final int FEDERATED_WORKER_PORT = 1222;
private final static int blocksize = 1024;
- private int rows = 10;
- private int cols = 10;
+ private final int rows = 10;
+ private final int cols = 10;
@Override
public void setUp() {
@@ -106,9 +106,6 @@ public class FederatedWorkerHandlerTest extends AutomatedTestBase {
try {
// we need the reference file to not be written to hdfs, so we get the correct format
rtplatform = Types.ExecMode.SINGLE_NODE;
- if (rtplatform == Types.ExecMode.SPARK) {
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
- }
programArgs = new String[] {"-w", Integer.toString(FEDERATED_WORKER_PORT)};
t = new Thread(() -> runTest(true, false, null, -1));
t.start();
@@ -155,7 +152,7 @@ public class FederatedWorkerHandlerTest extends AutomatedTestBase {
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
Types.ExecMode platformOld = rtplatform;
- Thread t = null;
+ Thread t;
getAndLoadTestConfiguration("aggregation");
String HOME = SCRIPT_DIR + TEST_DIR;
@@ -225,7 +222,7 @@ public class FederatedWorkerHandlerTest extends AutomatedTestBase {
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
Types.ExecMode platformOld = rtplatform;
- Thread t = null;
+ Thread t;
getAndLoadTestConfiguration("transfer");
String HOME = SCRIPT_DIR + TEST_DIR;
@@ -327,10 +324,10 @@ public class FederatedWorkerHandlerTest extends AutomatedTestBase {
fullDMLScriptName = HOME + MATVECMULT_TEST_NAME + ".dml";
programArgs = new String[] {"-checkPrivacy",
"-nvargs",
- "X1=" + TestUtils.federatedAddress("localhost", port1, input("X1")),
- "X2=" + TestUtils.federatedAddress("localhost", port2, input("X2")),
- "Y1=" + TestUtils.federatedAddress("localhost", port1, input("Y1")),
- "Y2=" + TestUtils.federatedAddress("localhost", port2, input("Y2")), "r=" + rows, "c=" + cols,
+ "X1=" + TestUtils.federatedAddress(port1, input("X1")),
+ "X2=" + TestUtils.federatedAddress(port2, input("X2")),
+ "Y1=" + TestUtils.federatedAddress(port1, input("Y1")),
+ "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols,
"hr=" + halfRows, "Z=" + output("Z")};
runTest(true, (expectedException != null), expectedException, -1);
diff --git a/src/test/scripts/functions/federated/FederatedFrameConstructionTest.dml b/src/test/scripts/functions/federated/FederatedFrameConstructionTest.dml
index df3078d..2b9bb34 100644
--- a/src/test/scripts/functions/federated/FederatedFrameConstructionTest.dml
+++ b/src/test/scripts/functions/federated/FederatedFrameConstructionTest.dml
@@ -19,5 +19,6 @@
#
#-------------------------------------------------------------
-A = federated(type="Frame", addresses=list($1, $1), ranges=list(list(0, 0), list($2, $3), list($2, 0), list($4, $3)))
-write(A, $5)
+A = federated(type="Frame", addresses=list($in, $in),
+ ranges=list(list(0, 0), list($rows, $cols), list($rows, 0), list($rows * 2, $cols)))
+write(A, $out)
diff --git a/src/test/scripts/functions/federated/FederatedL2SVMTest.dml b/src/test/scripts/functions/federated/FederatedL2SVMTest.dml
index 2ee4614..b5a4228 100644
--- a/src/test/scripts/functions/federated/FederatedL2SVMTest.dml
+++ b/src/test/scripts/functions/federated/FederatedL2SVMTest.dml
@@ -19,8 +19,8 @@
#
#-------------------------------------------------------------
-X = federated(addresses=list($1, $2),
- ranges=list(list(0, 0), list($5, $4), list($5, 0), list($3, $4)))
-Y = read($6)
+X = federated(addresses=list($in_X1, $in_X2),
+ ranges=list(list(0, 0), list($rows / 2, $cols), list($rows / 2, 0), list($rows, $cols)))
+Y = read($in_Y)
model = l2svm(X=X, Y=Y, intercept = FALSE, epsilon = 1e-12, lambda = 1, maxIterations = 100)
-write(model, $7)
+write(model, $out)
diff --git a/src/test/scripts/functions/federated/FederatedMatrixConstructionTest.dml b/src/test/scripts/functions/federated/FederatedMatrixConstructionTest.dml
index 840963b..de05d36 100644
--- a/src/test/scripts/functions/federated/FederatedMatrixConstructionTest.dml
+++ b/src/test/scripts/functions/federated/FederatedMatrixConstructionTest.dml
@@ -19,6 +19,6 @@
#
#-------------------------------------------------------------
-# TODO rework function definition
-A = federated(addresses=list($1, $1), ranges=list(list(0, 0), list($2, $3), list($2, 0), list($4, $3)))
-write(A, $5)
+A = federated(addresses=list($in, $in),
+ ranges=list(list(0, 0), list($rows, $cols), list($rows, 0), list($rows * 2, $cols)))
+write(A, $out)
diff --git a/src/test/scripts/functions/federated/FederatedMultiplyTest.dml b/src/test/scripts/functions/federated/FederatedMultiplyTest.dml
index 6acc84a..eff50a3 100644
--- a/src/test/scripts/functions/federated/FederatedMultiplyTest.dml
+++ b/src/test/scripts/functions/federated/FederatedMultiplyTest.dml
@@ -20,8 +20,8 @@
#-------------------------------------------------------------
X = federated(addresses=list($X1, $X2),
- ranges=list(list(0, 0), list($hr, $c), list($hr, 0), list($r, $c)))
+ ranges=list(list(0, 0), list($r / 2, $c), list($r / 2, 0), list($r, $c)))
Y = federated(addresses=list($Y1, $Y2),
- ranges=list(list(0, 0), list($c, $hr), list(0, $hr), list($c, $r)))
+ ranges=list(list(0, 0), list($c, $r / 2), list(0, $r / 2), list($c, $r)))
Z = X %*% Y
write(Z, $Z)
diff --git a/src/test/scripts/functions/federated/FederatedRCBindTest.dml b/src/test/scripts/functions/federated/FederatedRCBindTest.dml
index 0a7a425..1084f8c 100644
--- a/src/test/scripts/functions/federated/FederatedRCBindTest.dml
+++ b/src/test/scripts/functions/federated/FederatedRCBindTest.dml
@@ -19,9 +19,9 @@
#
#-------------------------------------------------------------
-A = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
-B = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
+A = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
+B = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
R = rbind(A, B)
C = cbind(A, B)
-write(R, $4)
-write(C, $5)
+write(R, $out_R)
+write(C, $out_C)
diff --git a/src/test/scripts/functions/federated/FederatedSumTest.dml b/src/test/scripts/functions/federated/FederatedSumTest.dml
index 09f28ea..8a8efb3 100644
--- a/src/test/scripts/functions/federated/FederatedSumTest.dml
+++ b/src/test/scripts/functions/federated/FederatedSumTest.dml
@@ -19,10 +19,10 @@
#
#-------------------------------------------------------------
-A = federated(addresses=list($1, $1), ranges=list(list(0, 0), list($2, $3), list($2, 0), list($4, $3)))
+A = federated(addresses=list($in, $in), ranges=list(list(0, 0), list($rows / 2, $cols), list($rows / 2, 0), list($rows, $cols)))
s = sum(A)
r = rowSums(A)
c = colSums(A)
-write(s, $5)
-write(r, $6)
-write(c, $7)
+write(s, $out_S)
+write(r, $out_R)
+write(c, $out_C)
diff --git a/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixAdditionScalar.dml b/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixAdditionScalar.dml
index c2ef777..9468448 100644
--- a/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixAdditionScalar.dml
+++ b/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixAdditionScalar.dml
@@ -22,7 +22,7 @@
# junit test class: org.apache.sysds.test.integration.functions.federated.FederatedMatrixScalarOperationsTest.java
-M = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
-S = $4
+M = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
+S = $scalar
R = M + S
-write(R, $5)
+write(R, $out)
diff --git a/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixMultiplicationScalar.dml b/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixMultiplicationScalar.dml
index cef0de2..8d92765 100644
--- a/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixMultiplicationScalar.dml
+++ b/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixMultiplicationScalar.dml
@@ -22,7 +22,7 @@
# junit test class: org.apache.sysds.test.integration.functions.federated.FederatedMatrixScalarOperationsTest.java
-M = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
-S = $4
+M = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
+S = $scalar
R = M * S
-write(R, $5)
\ No newline at end of file
+write(R, $out)
diff --git a/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixSubtractionScalar.dml b/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixSubtractionScalar.dml
index dbf8af4..cf4e940 100644
--- a/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixSubtractionScalar.dml
+++ b/src/test/scripts/functions/federated/matrix_scalar/FederatedMatrixSubtractionScalar.dml
@@ -22,7 +22,7 @@
# junit test class: org.apache.sysds.test.integration.functions.federated.FederatedMatrixScalarOperationsTest.java
-M = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
-S = $4
+M = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
+S = $scalar
R = M - S
-write(R, $5)
\ No newline at end of file
+write(R, $out)
diff --git a/src/test/scripts/functions/federated/matrix_scalar/ScalarAdditionFederatedMatrix.dml b/src/test/scripts/functions/federated/matrix_scalar/FederatedScalarAdditionMatrix.dml
similarity index 90%
rename from src/test/scripts/functions/federated/matrix_scalar/ScalarAdditionFederatedMatrix.dml
rename to src/test/scripts/functions/federated/matrix_scalar/FederatedScalarAdditionMatrix.dml
index bf42315..4d8392d 100644
--- a/src/test/scripts/functions/federated/matrix_scalar/ScalarAdditionFederatedMatrix.dml
+++ b/src/test/scripts/functions/federated/matrix_scalar/FederatedScalarAdditionMatrix.dml
@@ -22,7 +22,7 @@
# junit test class: org.apache.sysds.test.integration.functions.federated.FederatedMatrixScalarOperationsTest.java
-M = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
-S = $4
+M = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
+S = $scalar
R = S + M
-write(R, $5)
\ No newline at end of file
+write(R, $out)
\ No newline at end of file
diff --git a/src/test/scripts/functions/federated/matrix_scalar/ScalarMultiplicationFederatedMatrix.dml b/src/test/scripts/functions/federated/matrix_scalar/FederatedScalarMultiplicationMatrix.dml
similarity index 90%
rename from src/test/scripts/functions/federated/matrix_scalar/ScalarMultiplicationFederatedMatrix.dml
rename to src/test/scripts/functions/federated/matrix_scalar/FederatedScalarMultiplicationMatrix.dml
index bdd5294..d6081d4 100644
--- a/src/test/scripts/functions/federated/matrix_scalar/ScalarMultiplicationFederatedMatrix.dml
+++ b/src/test/scripts/functions/federated/matrix_scalar/FederatedScalarMultiplicationMatrix.dml
@@ -22,7 +22,7 @@
# junit test class: org.apache.sysds.test.integration.functions.federated.FederatedMatrixScalarOperationsTest.java
-M = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
-S = $4
+M = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
+S = $scalar
R = S * M
-write(R, $5)
\ No newline at end of file
+write(R, $out)
diff --git a/src/test/scripts/functions/federated/matrix_scalar/ScalarSubtractionFederatedMatrix.dml b/src/test/scripts/functions/federated/matrix_scalar/FederatedScalarSubtractionMatrix.dml
similarity index 90%
rename from src/test/scripts/functions/federated/matrix_scalar/ScalarSubtractionFederatedMatrix.dml
rename to src/test/scripts/functions/federated/matrix_scalar/FederatedScalarSubtractionMatrix.dml
index 2701f42..0b8fd00 100644
--- a/src/test/scripts/functions/federated/matrix_scalar/ScalarSubtractionFederatedMatrix.dml
+++ b/src/test/scripts/functions/federated/matrix_scalar/FederatedScalarSubtractionMatrix.dml
@@ -22,7 +22,7 @@
# junit test class: org.apache.sysds.test.integration.functions.federated.FederatedMatrixScalarOperationsTest.java
-M = federated(addresses=list($1), ranges=list(list(0, 0), list($2, $3)))
-S = $4
+M = federated(addresses=list($in), ranges=list(list(0, 0), list($rows, $cols)))
+S = $scalar
R = S - M
-write(R, $5)
\ No newline at end of file
+write(R, $out)