You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/04/20 08:22:16 UTC
[systemds] branch main updated: [MINOR] Fix rmempty tests and federated
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new f4ba2301f6 [MINOR] Fix rmempty tests and federated
f4ba2301f6 is described below
commit f4ba2301f6ad2550e2946f8a6296a36a72b25f4d
Author: OlgaOvcharenko <ov...@gmail.com>
AuthorDate: Wed Apr 20 01:33:22 2022 +0200
[MINOR] Fix rmempty tests and federated
This commit fixes both the rm empty test, and federated sparse rm empty.
Closes #1590
Closes #1587
---
.../fed/ParameterizedBuiltinFEDInstruction.java | 5 +-
src/test/java/org/apache/sysds/test/TestUtils.java | 14 ++
.../test/component/frame/FrameRemoveEmptyTest.java | 162 +++++++++++++++------
.../primitives/FederatedRemoveEmptyTest.java | 31 ++--
.../transform/TransformFederatedEncodeApply.dml | 1 -
5 files changed, 154 insertions(+), 59 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/fed/ParameterizedBuiltinFEDInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/fed/ParameterizedBuiltinFEDInstruction.java
index edae75ed39..2a1335426f 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/fed/ParameterizedBuiltinFEDInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/fed/ParameterizedBuiltinFEDInstruction.java
@@ -906,9 +906,8 @@ public class ParameterizedBuiltinFEDInstruction extends ComputationFEDInstructio
@Override
public FederatedResponse execute(ExecutionContext ec, Data... data) {
MatrixBlock mb = ((MatrixObject) data[0]).acquireReadAndRelease();
- int r = mb.getDenseBlockValues() != null ? mb.getNumRows() : 0;
- int c = mb.getDenseBlockValues() != null ? mb.getNumColumns() : 0;
- return new FederatedResponse(ResponseType.SUCCESS, new int[] {r, c});
+ final int[] dims = mb.isEmpty() ? new int[] {0, 0} : new int[] {mb.getNumRows(), mb.getNumColumns()};
+ return new FederatedResponse(ResponseType.SUCCESS, dims);
}
@Override
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java
index 4729e78bf0..ace1c10726 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -753,6 +753,11 @@ public class TestUtils
public static void compareMatrices(double[][] expectedMatrix, double[][] actualMatrix, int rows, int cols,
double epsilon, String message) {
+ if(expectedMatrix.length != rows && expectedMatrix[0].length != cols)
+ fail("Invalid number of rows and cols in expected");
+ if(actualMatrix.length != rows && actualMatrix[0].length != cols)
+ fail("Invalid number of rows and cols in actual");
+
int countErrors = 0;
for (int i = 0; i < rows && countErrors < 50; i++) {
for (int j = 0; j < cols && countErrors < 50; j++) {
@@ -1278,6 +1283,15 @@ public class TestUtils
compareMatrices(ret1, ret2, m2.getNumRows(), m2.getNumColumns(), tolerance, message);
}
+ public static void compareMatrices(MatrixBlock m1, double[][] m2, double tolerance, String message) {
+ double[][] ret1 = DataConverter.convertToDoubleMatrix(m1);
+ compareMatrices(ret1, m2, m1.getNumRows(), m1.getNumColumns(), tolerance, message);
+ }
+
+ public static void compareMatrices(double[][] m1, MatrixBlock m2, double tolerance, String message) {
+ double[][] ret2 = DataConverter.convertToDoubleMatrix(m2);
+ compareMatrices(m1, ret2, m2.getNumRows(), m2.getNumColumns(), tolerance, message);
+ }
/**
* Compares two matrices given as HashMaps. The matrix containing more nnz
diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameRemoveEmptyTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameRemoveEmptyTest.java
index 5f011da1a7..5fb0913c65 100644
--- a/src/test/java/org/apache/sysds/test/component/frame/FrameRemoveEmptyTest.java
+++ b/src/test/java/org/apache/sysds/test/component/frame/FrameRemoveEmptyTest.java
@@ -19,8 +19,12 @@
package org.apache.sysds.test.component.frame;
+import static org.junit.Assert.fail;
+
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.api.DMLScript;
import org.apache.sysds.common.Types;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
@@ -32,51 +36,110 @@ import org.apache.sysds.test.functions.unary.matrix.RemoveEmptyTest;
import org.junit.Test;
public class FrameRemoveEmptyTest extends AutomatedTestBase {
+
+ private static final Log LOG = LogFactory.getLog(FrameRemoveEmptyTest.class.getName());
+
private final static String TEST_NAME1 = "removeEmpty1";
private final static String TEST_NAME2 = "removeEmpty2";
private final static String TEST_DIR = "functions/frame/";
private static final String TEST_CLASS_DIR = TEST_DIR + RemoveEmptyTest.class.getSimpleName() + "/";
- private final static int _rows = 10;
- private final static int _cols = 6;
-
- private final static double _sparsityDense = 0.7;
+ private final static double _dense = 0.99;
+ private final static double _sparse = 0.1;
@Override
public void setUp() {
- addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"V"}));
- addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"V"}));
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"}));
+ addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"}));
}
@Test
public void testRemoveEmptyRowsCP() {
- runTestRemoveEmpty(TEST_NAME1, "rows", Types.ExecType.CP, false, false);
+ runTestRemoveEmpty(TEST_NAME1, "rows", Types.ExecType.CP, false, false, 100, 100, _dense);
+ }
+
+ @Test
+ public void testRemoveEmptyRowsCPSparse() {
+ runTestRemoveEmpty(TEST_NAME1, "rows", Types.ExecType.CP, false, false, 100, 100, _sparse);
+ }
+
+ @Test
+ public void testRemoveEmptyRowsCPSparse2() {
+ runTestRemoveEmpty(TEST_NAME1, "rows", Types.ExecType.CP, false, false, 1000, 10, _sparse);
}
@Test
public void testRemoveEmptyColsCP() {
- runTestRemoveEmpty(TEST_NAME1, "cols", Types.ExecType.CP, false, false);
+ runTestRemoveEmpty(TEST_NAME1, "cols", Types.ExecType.CP, false, false, 100, 100, _dense);
+ }
+
+ @Test
+ public void testRemoveEmptyColsCPSparse() {
+ runTestRemoveEmpty(TEST_NAME1, "cols", Types.ExecType.CP, false, false, 100, 100, _sparse);
+ }
+
+ @Test
+ public void testRemoveEmptyColsCPSparse2() {
+ runTestRemoveEmpty(TEST_NAME1, "cols", Types.ExecType.CP, false, false, 10, 1000, _sparse);
}
@Test
public void testRemoveEmptyRowsSelectFullCP() {
- runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, true);
+ runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, true, 100, 100, _dense);
+ }
+
+ @Test
+ public void testRemoveEmptyRowsSelectFullCPSparse() {
+ runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, true, 100, 100, _sparse);
+ }
+
+ @Test
+ public void testRemoveEmptyRowsSelectFullCPSparse2() {
+ runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, true, 100, 10, _sparse);
}
@Test
- public void testRemoveEmptyColsSelectFullCP() { runTestRemoveEmpty(TEST_NAME2, "cols", Types.ExecType.CP, true, true); }
+ public void testRemoveEmptyColsSelectFullCP() {
+ runTestRemoveEmpty(TEST_NAME2, "cols", Types.ExecType.CP, true, true, 100, 100, _dense);
+ }
+
+ @Test
+ public void testRemoveEmptyColsSelectFullCPSparse() {
+ runTestRemoveEmpty(TEST_NAME2, "cols", Types.ExecType.CP, true, true, 100, 100, _sparse);
+ }
@Test
public void testRemoveEmptyRowsSelectCP() {
- runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, false);
+ runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, false, 100, 100, _dense);
+ }
+
+ @Test
+ public void testRemoveEmptyRowsSelectCPSparse() {
+ runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, false, 100, 100, _sparse);
+ }
+
+ @Test
+ public void testRemoveEmptyRowsSelectCPSparse2() {
+ runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, false, 100, 10, _sparse);
+ }
+
+ @Test
+ public void testRemoveEmptyRowsSelectCPSparse3() {
+ runTestRemoveEmpty(TEST_NAME2, "rows", Types.ExecType.CP, true, false, 100, 3, _sparse);
}
@Test
public void testRemoveEmptyColsSelectCP() {
- runTestRemoveEmpty(TEST_NAME2, "cols", Types.ExecType.CP, true, false);
+ runTestRemoveEmpty(TEST_NAME2, "cols", Types.ExecType.CP, true, false, 100, 100, _dense);
+ }
+
+ @Test
+ public void testRemoveEmptyColsSelectCPSparse() {
+ runTestRemoveEmpty(TEST_NAME2, "cols", Types.ExecType.CP, true, false, 100, 100, _sparse);
}
- private void runTestRemoveEmpty(String testname, String margin, Types.ExecType et, boolean bSelectIndex, boolean fullSelect) {
+ private void runTestRemoveEmpty(String testname, String margin, Types.ExecType et, boolean bSelectIndex,
+ boolean fullSelect, int rows, int cols, double sparsity) {
Types.ExecMode platformOld = rtplatform;
switch(et) {
case SPARK:
@@ -94,27 +157,33 @@ public class FrameRemoveEmptyTest extends AutomatedTestBase {
try {
// register test configuration
TestConfiguration config = getTestConfiguration(testname);
- config.addVariable("rows", _rows);
- config.addVariable("cols", _cols);
+ config.addVariable("rows", rows);
+ config.addVariable("cols", cols);
loadTestConfiguration(config);
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[] {"-explain", "-args", input("V"), input("I"), margin, output("V")};
+ programArgs = new String[] {"-explain", "-args", input("V"), input("I"), margin, output("R")};
+
+ Pair<MatrixBlock, MatrixBlock> data = createInputMatrix(margin, bSelectIndex, fullSelect, rows, cols, sparsity);
- Pair<MatrixBlock, MatrixBlock> data = createInputMatrix(margin, bSelectIndex, fullSelect);
MatrixBlock in = data.getKey();
MatrixBlock select = data.getValue();
- runTest(true, false, null, -1);
+ runTest(null);
- double[][] outArray = TestUtils.convertHashMapToDoubleArray(readDMLMatrixFromOutputDir("V"));
- MatrixBlock out = new MatrixBlock(outArray.length, outArray[0].length, false);
- out.init(outArray, outArray.length, outArray[0].length);
+ MatrixBlock expected = fullSelect ? in : in.removeEmptyOperations(new MatrixBlock(), margin.equals("rows"),
+ false, select);
- MatrixBlock expected = fullSelect ? in :
- in.removeEmptyOperations(new MatrixBlock(), margin.equals("rows"), false, select);
- TestUtils.compareMatrices(expected, out, 0);
+ double[][] out = TestUtils.convertHashMapToDoubleArray(readDMLMatrixFromOutputDir("R"));
+
+ LOG.debug(expected.getNumRows() + " " + out.length);
+
+ TestUtils.compareMatrices(expected, out, 0, "");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("Failed test because of exception " + e);
}
finally {
// reset platform for additional tests
@@ -123,37 +192,37 @@ public class FrameRemoveEmptyTest extends AutomatedTestBase {
}
}
- private Pair<MatrixBlock, MatrixBlock> createInputMatrix(String margin, boolean bSelectIndex, boolean fullSelect) {
+ private Pair<MatrixBlock, MatrixBlock> createInputMatrix(String margin, boolean bSelectIndex, boolean fullSelect,
+ int rows, int cols, double sparsity) {
int rowsp = -1, colsp = -1;
if(margin.equals("rows")) {
- rowsp = _rows / 2;
- colsp = _cols;
+ rowsp = rows / 2;
+ colsp = cols;
}
else {
- rowsp = _rows;
- colsp = _cols / 2;
+ rowsp = rows;
+ colsp = cols / 2;
}
// long seed = System.nanoTime();
- double[][] V = getRandomMatrix(_rows, _cols, 0, 1,
- FrameRemoveEmptyTest._sparsityDense, 7);
+ double[][] V = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
double[][] Vp = new double[rowsp][colsp];
double[][] Ix;
int innz = 0, vnnz = 0;
// clear out every other row/column
if(margin.equals("rows")) {
- Ix = new double[_rows][1];
- for(int i = 0; i < _rows; i++) {
+ Ix = new double[rows][1];
+ for(int i = 0; i < rows; i++) {
boolean clear = i % 2 != 0;
- if(clear && !fullSelect) {
- for(int j = 0; j < _cols; j++)
+ if(clear && !fullSelect) {
+ for(int j = 0; j < cols; j++)
V[i][j] = 0;
Ix[i][0] = 0;
}
else {
boolean bNonEmpty = false;
- for(int j = 0; j < _cols; j++) {
+ for(int j = 0; j < cols; j++) {
Vp[i / 2][j] = V[i][j];
bNonEmpty |= V[i][j] != 0.0;
vnnz += (V[i][j] == 0.0) ? 0 : 1;
@@ -164,17 +233,17 @@ public class FrameRemoveEmptyTest extends AutomatedTestBase {
}
}
else {
- Ix = new double[1][_cols];
- for(int j = 0; j < _cols; j++) {
+ Ix = new double[1][cols];
+ for(int j = 0; j < cols; j++) {
boolean clear = j % 2 != 0;
if(clear && !fullSelect) {
- for(int i = 0; i < _rows; i++)
+ for(int i = 0; i < rows; i++)
V[i][j] = 0;
Ix[0][j] = 0;
}
else {
boolean bNonEmpty = false;
- for(int i = 0; i < _rows; i++) {
+ for(int i = 0; i < rows; i++) {
Vp[i][j / 2] = V[i][j];
bNonEmpty |= V[i][j] != 0.0;
vnnz += (V[i][j] == 0.0) ? 0 : 1;
@@ -185,12 +254,12 @@ public class FrameRemoveEmptyTest extends AutomatedTestBase {
}
}
- MatrixCharacteristics imc = new MatrixCharacteristics(margin.equals("rows") ? FrameRemoveEmptyTest._rows : 1,
- margin.equals("rows") ? 1 : _cols, 1000, innz);
- MatrixCharacteristics vmc = new MatrixCharacteristics(_rows, _cols, 1000, vnnz);
+ MatrixCharacteristics imc = new MatrixCharacteristics(margin.equals("rows") ? rows : 1,
+ margin.equals("rows") ? 1 : cols, 1000, innz);
+ MatrixCharacteristics vmc = new MatrixCharacteristics(rows, cols, 1000, vnnz);
- MatrixBlock in = new MatrixBlock(_rows, _cols, false);
- in.init(V, _rows, _cols);
+ MatrixBlock in = new MatrixBlock(rows, cols, false);
+ in.init(V, rows, cols);
MatrixBlock select = new MatrixBlock(Ix.length, Ix[0].length, false);
select.init(Ix, Ix.length, Ix[0].length);
@@ -200,6 +269,9 @@ public class FrameRemoveEmptyTest extends AutomatedTestBase {
if(bSelectIndex)
writeInputMatrixWithMTD("I", Ix, false, imc);
+ in.examSparsity();
+ select.examSparsity();
+
return new ImmutablePair<>(in, select);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRemoveEmptyTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRemoveEmptyTest.java
index af2b9bba84..3c3992f616 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRemoveEmptyTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRemoveEmptyTest.java
@@ -45,18 +45,30 @@ public class FederatedRemoveEmptyTest extends AutomatedTestBase {
private static final String TEST_CLASS_DIR = TEST_DIR + FederatedRemoveEmptyTest.class.getSimpleName() + "/";
private final static int blocksize = 1024;
+
@Parameterized.Parameter()
public int rows;
@Parameterized.Parameter(1)
public int cols;
@Parameterized.Parameter(2)
public boolean rowPartitioned;
+ @Parameterized.Parameter(3)
+ public double sparsity;
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
- {20, 12, true},
- {20, 12, false}
+ // dense
+ {20, 12, true, 1.0},
+ {20, 12, false, 1.0},
+ // sparse
+ {20, 12, true, 0.1},
+ {20, 12, false, 0.1},
+ {1000, 12, true, 0.1},
+ {1000, 12, false, 0.1},
+ {20, 1000, true, 0.1},
+ {20, 1000, false, 0.1},
+ {40, 40, true, 0} // empty
});
}
@@ -94,10 +106,10 @@ public class FederatedRemoveEmptyTest extends AutomatedTestBase {
c = cols;
}
- double[][] X1 = getRandomMatrix(r, c, 1, 5, 1, 3);
- double[][] X2 = getRandomMatrix(r, c, 1, 5, 1, 7);
- double[][] X3 = getRandomMatrix(r, c, 1, 5, 1, 8);
- double[][] X4 = getRandomMatrix(r, c, 1, 5, 1, 9);
+ double[][] X1 = getRandomMatrix(r, c, 1, 5, sparsity, 3);
+ double[][] X2 = getRandomMatrix(r, c, 1, 5, sparsity, 7);
+ double[][] X3 = getRandomMatrix(r, c, 1, 5, sparsity, 8);
+ double[][] X4 = getRandomMatrix(r, c, 1, 5, sparsity, 9);
for(int k : new int[] {1, 2, 3}) {
Arrays.fill(X3[k], 0);
@@ -121,10 +133,9 @@ public class FederatedRemoveEmptyTest extends AutomatedTestBase {
Thread t4 = startLocalFedWorkerThread(port4);
rtplatform = execMode;
- if(rtplatform == ExecMode.SPARK) {
- System.out.println(7);
+ if(rtplatform == ExecMode.SPARK)
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
- }
+
TestConfiguration config = availableTestConfigurations.get(TEST_NAME);
loadTestConfiguration(config);
@@ -160,4 +171,4 @@ public class FederatedRemoveEmptyTest extends AutomatedTestBase {
DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
}
-}
+}
\ No newline at end of file
diff --git a/src/test/scripts/functions/transform/TransformFederatedEncodeApply.dml b/src/test/scripts/functions/transform/TransformFederatedEncodeApply.dml
index 4291993e8f..6305040e85 100644
--- a/src/test/scripts/functions/transform/TransformFederatedEncodeApply.dml
+++ b/src/test/scripts/functions/transform/TransformFederatedEncodeApply.dml
@@ -36,4 +36,3 @@ X2 = transformapply(target=F1, spec=jspec, meta=M);
write(X, $TFDATA1, format="csv");
write(X2, $TFDATA2, format="csv");
-