You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/08/07 03:45:31 UTC
systemml git commit: [SYSTEMML-1826] Fix scalar-frame casts and frame
rbind operations
Repository: systemml
Updated Branches:
refs/heads/master 9da5eab00 -> 70e5f29e8
[SYSTEMML-1826] Fix scalar-frame casts and frame rbind operations
This patch fixes the missing compiler support for scalar-frame casting
as well as missing meta data handling on frame rbind operations. This
also includes a suite of related testcases and some minor cleanups in
AutomatedTestBase (base class of all tests).
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/70e5f29e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/70e5f29e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/70e5f29e
Branch: refs/heads/master
Commit: 70e5f29e8fff0e698d7abc647efe3f79be38abba
Parents: 9da5eab
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Aug 6 20:45:51 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Aug 6 20:46:16 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/hops/UnaryOp.java | 6 +-
.../sysml/runtime/matrix/data/FrameBlock.java | 9 +-
.../test/integration/AutomatedTestBase.java | 4 -
.../frame/FrameScalarCastingIntegratedTest.java | 171 +++++++++++++++++++
.../scripts/functions/frame/FrameScalarCast.dml | 30 ++++
.../functions/frame/ZPackageSuite.java | 1 +
6 files changed, 211 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 35902b7..2b31247 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -618,8 +618,7 @@ public class UnaryOp extends Hop implements MultiThreadedHop
|| _op == OpOp1.CUMMAX );
}
- public boolean isCastUnaryOperation()
- {
+ public boolean isCastUnaryOperation() {
return ( _op == OpOp1.CAST_AS_MATRIX
|| _op == OpOp1.CAST_AS_SCALAR
|| _op == OpOp1.CAST_AS_FRAME
@@ -695,7 +694,8 @@ public class UnaryOp extends Hop implements MultiThreadedHop
{
//do nothing always known
}
- else if( _op == OpOp1.CAST_AS_MATRIX && getInput().get(0).getDataType()==DataType.SCALAR )
+ else if( (_op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_FRAME)
+ && getInput().get(0).getDataType()==DataType.SCALAR )
{
//prevent propagating 0 from scalar (which would be interpreted as unknown)
setDim1( 1 );
http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 10b39fe..45ad26c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -967,7 +967,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
//concatenate column data (w/ deep copy to prevent side effects)
ret._coldata = (Array[]) ArrayUtils.addAll(_coldata, that._coldata);
- for( int i=0; i<ret._coldata.length; i++ )
+ for( int i=0; i<ret.getNumColumns(); i++ )
ret._coldata[i] = ret._coldata[i].clone();
}
else //ROW APPEND
@@ -984,10 +984,13 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
ret._numRows = _numRows;
ret._schema = _schema.clone();
ret._colnames = (_colnames!=null) ? _colnames.clone() : null;
+ ret._colmeta = new ColumnMetadata[getNumColumns()];
+ for( int j=0; j<_schema.length; j++ )
+ ret._colmeta[j] = new ColumnMetadata(0);
//concatenate data (deep copy first, append second)
- ret._coldata = new Array[_coldata.length];
- for( int j=0; j<_coldata.length; j++ )
+ ret._coldata = new Array[getNumColumns()];
+ for( int j=0; j<getNumColumns(); j++ )
ret._coldata[j] = _coldata[j].clone();
Iterator<Object[]> iter = that.getObjectRowIterator();
while( iter.hasNext() )
http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index 5a1b904..07802dd 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -195,8 +195,6 @@ public abstract class AutomatedTestBase
protected static RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID;
protected static final boolean DEBUG = false;
- protected static final boolean VISUALIZE = false;
- protected static final boolean RUNNETEZZA = false;
protected String fullDMLScriptName; // utilize for both DML and PyDML, should probably be renamed.
// protected String fullPYDMLScriptName;
@@ -1174,8 +1172,6 @@ public abstract class AutomatedTestBase
}
}
// program-independent parameters
- if(VISUALIZE)
- args.add("-v");
args.add("-exec");
if(rtplatform == RUNTIME_PLATFORM.HADOOP)
args.add("hadoop");
http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java
new file mode 100644
index 0000000..7302126
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
+
+public class FrameScalarCastingIntegratedTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/frame/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + FrameScalarCastingIntegratedTest.class.getSimpleName() + "/";
+ private final static String TEST_NAME = "FrameScalarCast";
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+ }
+
+ @Test
+ public void testFrameStringCP0() {
+ runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.SINGLE_NODE);
+ }
+
+ @Test
+ public void testFrameLongCP0() {
+ runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.SINGLE_NODE);
+ }
+
+ @Test
+ public void testFrameBooleanCP0() {
+ runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.SINGLE_NODE);
+ }
+
+ @Test
+ public void testFrameDoubleCP0() {
+ runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.SINGLE_NODE);
+ }
+
+ @Test
+ public void testFrameStringCP1() {
+ runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.HYBRID);
+ }
+
+ @Test
+ public void testFrameLongCP1() {
+ runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.HYBRID);
+ }
+
+ @Test
+ public void testFrameBooleanCP1() {
+ runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.HYBRID);
+ }
+
+ @Test
+ public void testFrameDoubleCP1() {
+ runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.HYBRID);
+ }
+
+ @Test
+ public void testFrameStringCP2() {
+ runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.HYBRID_SPARK);
+ }
+
+ @Test
+ public void testFrameLongCP2() {
+ runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.HYBRID_SPARK);
+ }
+
+ @Test
+ public void testFrameBooleanCP2() {
+ runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.HYBRID_SPARK);
+ }
+
+ @Test
+ public void testFrameDoubleCP2() {
+ runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.HYBRID_SPARK);
+ }
+
+ @Test
+ public void testFrameStringSP() {
+ runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.SPARK);
+ }
+
+ @Test
+ public void testFrameLongSP() {
+ runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.SPARK);
+ }
+
+ @Test
+ public void testFrameBooleanSP() {
+ runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.SPARK);
+ }
+
+ @Test
+ public void testFrameDoubleSP() {
+ runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.SPARK);
+ }
+
+ private void runFrameScalarCastingTest(ValueType vtIn, RUNTIME_PLATFORM et)
+ {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ rtplatform = et;
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ getAndLoadTestConfiguration(TEST_NAME);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args", input("V"), output("R") };
+
+ //generate input data
+ switch( vtIn ) {
+ case STRING: MapReduceTool.writeStringToHDFS("foo", input("V")); break;
+ case INT: MapReduceTool.writeIntToHDFS(7, input("V")); break;
+ case BOOLEAN: MapReduceTool.writeBooleanToHDFS(true, input("V")); break;
+ case DOUBLE: MapReduceTool.writeDoubleToHDFS(7.3, input("V")); break;
+ default: throw new RuntimeException("Unsupported type: "+vtIn);
+ }
+ MapReduceTool.writeScalarMetaDataFile(input("V")+".mtd", vtIn);
+
+ //run tests
+ runTest(true, false, null, -1);
+
+ //compare output
+ Assert.assertEquals(readDMLMatrixFromHDFS("R").get(new CellIndex(1,1)), Double.valueOf(1));
+ if( et != RUNTIME_PLATFORM.SPARK ) {
+ Assert.assertTrue(Statistics.getNoOfCompiledSPInst()==0);
+ Assert.assertTrue(Statistics.getNoOfExecutedSPInst()==0);
+ }
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/scripts/functions/frame/FrameScalarCast.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/FrameScalarCast.dml b/src/test/scripts/functions/frame/FrameScalarCast.dml
new file mode 100644
index 0000000..25d9bd2
--- /dev/null
+++ b/src/test/scripts/functions/frame/FrameScalarCast.dml
@@ -0,0 +1,30 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+s1 = read($1);
+
+F1 = as.frame(s1);
+F2 = rbind(F1, F1);
+s2 = as.scalar(F2[1,1])
+R = as.matrix(s1==s2);
+
+write(R, $2);
http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
index 50d7a67..45617ba 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
@@ -41,6 +41,7 @@ import org.junit.runners.Suite;
FrameMatrixWriteTest.class,
FrameMetaReadWriteTest.class,
FrameReadWriteTest.class,
+ FrameScalarCastingIntegratedTest.class,
FrameScalarCastingTest.class,
FrameSchemaReadTest.class,
FrameSerializationTest.class,