You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/09/11 22:20:09 UTC
[2/2] systemml git commit: [SYSTEMML-1901] Fix parfor support for
frame inputs and intermediates
[SYSTEMML-1901] Fix parfor support for frame inputs and intermediates
This patch fixes parfor optimizer and runtime issues regarding the
support of frame inputs and intermediates. Accordingly, this patch also
adds some related tests to avoid such issues in the future.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/75454819
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/75454819
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/75454819
Branch: refs/heads/master
Commit: 754548190a7ade052ab7218941bd2fa43b50b318
Parents: f45493e
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Sep 11 15:19:00 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Sep 11 15:20:06 2017 -0700
----------------------------------------------------------------------
.../controlprogram/ParForProgramBlock.java | 3 +-
.../parfor/opt/OptimizerRuleBased.java | 13 ++-
.../frame/ParforFrameIntermediateTest.java | 97 ++++++++++++++++++++
.../frame/ParforFrameIntermediates.dml | 29 ++++++
.../functions/frame/ZPackageSuite.java | 1 +
5 files changed, 135 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index b393cd1..4dda1b4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -1279,7 +1279,8 @@ public class ParForProgramBlock extends ForProgramBlock
}
break;
case MATRIX:
- //currently we do not create any unscoped matrix object outputs
+ case FRAME:
+ //currently we do not create any unscoped matrix or frame outputs
//because metadata (e.g., outputinfo) not known at this place.
break;
case UNKNOWN:
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index ec229d5..72b9566 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -456,26 +456,25 @@ public class OptimizerRuleBased extends Optimizer
{
boolean ret = false;
- if( !n.isLeaf() )
- {
+ if( !n.isLeaf() ) {
for( OptNode cn : n.getChilds() )
if( cn.getNodeType() != NodeType.FUNCCALL ) //prevent conflicts with aliases
ret |= rFindDataPartitioningCandidates( cn, cand, vars, thetaM );
}
else if( n.getNodeType()== NodeType.HOP
- && n.getParam(ParamType.OPSTRING).equals(IndexingOp.OPSTRING) )
+ && n.getParam(ParamType.OPSTRING).equals(IndexingOp.OPSTRING) )
{
Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
String inMatrix = h.getInput().get(0).getName();
- if( cand.containsKey(inMatrix) ) //Required Condition: partitioning applicable
+ if( cand.containsKey(inMatrix) && h.getDataType().isMatrix() ) //Required: partitionable
{
PartitionFormat dpf = cand.get(inMatrix);
double mnew = getNewRIXMemoryEstimate( n, inMatrix, dpf, vars );
//NOTE: for the moment, we do not partition according to the remote mem, because we can execute
- //it even without partitioning in CP. However, advanced optimizers should reason about this
+ //it even without partitioning in CP. However, advanced optimizers should reason about this
//double mold = h.getMemEstimate();
- if( n.getExecType() == getRemoteExecType() //Opt Condition: MR/Spark
- || h.getMemEstimate() > thetaM ) //Opt Condition: mem estimate > constraint to force partitioning
+ if( n.getExecType() == getRemoteExecType() //Opt Condition: MR/Spark
+ || h.getMemEstimate() > thetaM ) //Opt Condition: mem estimate > constraint to force partitioning
{
//NOTE: subsequent rewrites will still use the MR mem estimate
//(guarded by subsequent operations that have at least the memory req of one partition)
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java
new file mode 100644
index 0000000..c9ac763
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.io.FrameWriterFactory;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+public class ParforFrameIntermediateTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/frame/";
+ private final static String TEST_NAME = "ParforFrameIntermediates";
+ private final static String TEST_CLASS_DIR = TEST_DIR + ParforFrameIntermediateTest.class.getSimpleName() + "/";
+
+ private final static int rows = 1382;
+ private final static int cols = 5;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"F2"}));
+ }
+
+ @Test
+ public void testParforFrameIntermediatesCP() {
+ runParforFrameIntermediatesTest(ExecType.CP);
+ }
+
+ @Test
+ public void testParforFrameIntermediatesSpark() {
+ runParforFrameIntermediatesTest(ExecType.SPARK);
+ }
+
+ private void runParforFrameIntermediatesTest( ExecType et ) {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ){
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK
+ || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ //setup testcase
+ getAndLoadTestConfiguration(TEST_NAME);
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-explain", "-args", input("F")};
+
+ //generate input data and write as frame
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, 0.9, 8362);
+ FrameBlock fA = DataConverter.convertToFrameBlock(
+ DataConverter.convertToMatrixBlock(A));
+ FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo)
+ .writeFrameToHDFS(fA, input("F"), rows, cols);
+
+ //run test
+ runTest(true, false, null, -1);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/test/scripts/functions/frame/ParforFrameIntermediates.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/ParforFrameIntermediates.dml b/src/test/scripts/functions/frame/ParforFrameIntermediates.dml
new file mode 100644
index 0000000..1fee376
--- /dev/null
+++ b/src/test/scripts/functions/frame/ParforFrameIntermediates.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+F = read($1, data_type="frame", format="csv");
+
+parfor(i in 1:ncol(F)) {
+ Fi = F[, i];
+ Mi = as.matrix(Fi) + 7;
+ Fi2 = as.frame(Mi);
+ print(toString(Fi2[1:2,]));
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
index 45617ba..bbdb9d2 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
@@ -45,6 +45,7 @@ import org.junit.runners.Suite;
FrameScalarCastingTest.class,
FrameSchemaReadTest.class,
FrameSerializationTest.class,
+ ParforFrameIntermediateTest.class,
})