You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/10/04 18:35:11 UTC
[2/2] incubator-systemml git commit: [SYSTEMML-1008] Fix removeEmpty
w/ select vector (empty input handling)
[SYSTEMML-1008] Fix removeEmpty w/ select vector (empty input handling)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/4e0949ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/4e0949ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/4e0949ba
Branch: refs/heads/master
Commit: 4e0949bae4724987d9486449796bdf6675e11f4a
Parents: 8da7009
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Mon Oct 3 18:47:11 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Mon Oct 3 18:47:11 2016 -0700
----------------------------------------------------------------------
.../runtime/matrix/data/LibMatrixReorg.java | 8 +-
.../unary/matrix/RemoveEmptySelTest.java | 161 +++++++++++++++++++
.../functions/unary/matrix/removeEmptySel.dml | 35 ++++
.../functions/unary/matrix/ZPackageSuite.java | 1 +
4 files changed, 203 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4e0949ba/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
index 179f835..af30af8 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
@@ -596,7 +596,7 @@ public class LibMatrixReorg
//check for empty inputs
//(the semantics of removeEmpty are that for an empty m-by-n matrix, the output
//is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0)
- if( in.isEmptyBlock(false) ) {
+ if( in.isEmptyBlock(false) && select == null ) {
if( rows )
ret.reset(1, in.clen, in.sparse);
else //cols
@@ -1947,6 +1947,8 @@ public class LibMatrixReorg
rlen2 = Math.max(rlen2, 1); //ensure valid output
boolean sp = MatrixBlock.evalSparseFormatInMemory(rlen2, n, in.nonZeros);
ret.reset(rlen2, n, sp);
+ if( in.isEmptyBlock(false) )
+ return ret;
if( in.sparse ) //* <- SPARSE
{
@@ -2055,7 +2057,9 @@ public class LibMatrixReorg
clen2 = Math.max(clen2, 1); //ensure valid output
boolean sp = MatrixBlock.evalSparseFormatInMemory(m, clen2, in.nonZeros);
ret.reset(m, clen2, sp);
-
+ if( in.isEmptyBlock(false) )
+ return ret;
+
if( in.sparse ) //* <- SPARSE
{
//note: output dense or sparse
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4e0949ba/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptySelTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptySelTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptySelTest.java
new file mode 100644
index 0000000..cba0e99
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/RemoveEmptySelTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.unary.matrix;
+
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class RemoveEmptySelTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "removeEmptySel";
+ private final static String TEST_DIR = "functions/unary/matrix/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + RemoveEmptySelTest.class.getSimpleName() + "/";
+
+ private final static int rows = 1007;
+ private final static int cols = 1005;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "R" }) );
+ }
+
+ @Test
+ public void testRemoveEmptyRowSelCP() {
+ runTestRemoveEmptySel( "rows", ExecType.CP, false );
+ }
+
+ @Test
+ public void testRemoveEmptyRowSelEmptyCP() {
+ runTestRemoveEmptySel( "rows", ExecType.CP, true );
+ }
+
+ @Test
+ public void testRemoveEmptyColSelCP() {
+ runTestRemoveEmptySel( "cols", ExecType.CP, false );
+ }
+
+ @Test
+ public void testRemoveEmptyColSelEmptyCP() {
+ runTestRemoveEmptySel( "cols", ExecType.CP, true );
+ }
+
+ @Test
+ public void testRemoveEmptyRowSelSP() {
+ runTestRemoveEmptySel( "rows", ExecType.SPARK, false );
+ }
+
+ @Test
+ public void testRemoveEmptyRowSelEmptySP() {
+ runTestRemoveEmptySel( "rows", ExecType.SPARK, true );
+ }
+
+ @Test
+ public void testRemoveEmptyColSelSP() {
+ runTestRemoveEmptySel( "cols", ExecType.SPARK, false );
+ }
+
+ @Test
+ public void testRemoveEmptyColSelEmptySP() {
+ runTestRemoveEmptySel( "cols", ExecType.SPARK, true );
+ }
+
+ @Test
+ public void testRemoveEmptyRowSelMR() {
+ runTestRemoveEmptySel( "rows", ExecType.MR, false );
+ }
+
+ @Test
+ public void testRemoveEmptyRowSelEmptyMR() {
+ runTestRemoveEmptySel( "rows", ExecType.MR, true );
+ }
+
+ @Test
+ public void testRemoveEmptyColSelMR() {
+ runTestRemoveEmptySel( "cols", ExecType.MR, false );
+ }
+
+ @Test
+ public void testRemoveEmptyColSelEmptyMR() {
+ runTestRemoveEmptySel( "cols", ExecType.MR, true );
+ }
+
+ /**
+ *
+ * @param testname
+ * @param margin
+ * @param et
+ * @param sparse
+ * @param empty
+ */
+ private void runTestRemoveEmptySel( String margin, ExecType et, boolean empty )
+ {
+ //rtplatform for MR
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ){
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ //register test configuration
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ config.addVariable("rows", rows);
+ config.addVariable("cols", cols);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args", Integer.toString(empty?0:1),
+ Integer.toString(rows), Integer.toString(cols), margin, output("R") };
+
+ //run dml test
+ runTest(true, false, null, -1);
+
+ //compare expected dimensions
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ Double expectedRows = (double) (margin.equals("rows") ? rows/2 : rows);
+ Double expectedCols = (double) (margin.equals("cols") ? cols/2 : cols);
+ Assert.assertEquals("Wrong output nrow.", expectedRows, dmlfile.get(new CellIndex(1,1)));
+ Assert.assertEquals("Wrong output ncol.", expectedCols, dmlfile.get(new CellIndex(2,1)));
+ }
+ finally
+ {
+ //reset platform for additional tests
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4e0949ba/src/test/scripts/functions/unary/matrix/removeEmptySel.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/removeEmptySel.dml b/src/test/scripts/functions/unary/matrix/removeEmptySel.dml
new file mode 100644
index 0000000..297914e
--- /dev/null
+++ b/src/test/scripts/functions/unary/matrix/removeEmptySel.dml
@@ -0,0 +1,35 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+V = matrix($1, rows=$2, cols=$3);
+if( $4=="rows" ) {
+ S = matrix(0, rows=nrow(V), cols=1)
+ S[1:(nrow(V)/2),] = matrix(1, rows=nrow(V)/2, cols=1);
+}
+else {
+ S = matrix(0, rows=1, cols=ncol(V))
+ S[,1:(ncol(V)/2)] = matrix(1, rows=1, cols=ncol(V)/2);
+}
+
+Vp = removeEmpty(target=V, margin=$4, select=S);
+R = rbind(as.matrix(nrow(Vp)), as.matrix(ncol(Vp)));
+write(R, $5);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4e0949ba/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
index dac067d..65bc06b 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
@@ -48,6 +48,7 @@ import org.junit.runners.Suite;
NegationTest.class,
PrintTest.class,
QRSolverTest.class,
+ RemoveEmptySelTest.class,
RemoveEmptyTest.class,
ReplaceTest.class,
RoundTest.class,