You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/09/11 22:20:08 UTC
[1/2] systemml git commit: [SYSTEMML-1900] Allow external codegen
java compiler configuration
Repository: systemml
Updated Branches:
refs/heads/master 687e19c55 -> 754548190
[SYSTEMML-1900] Allow external codegen java compiler configuration
This patch exposes the codegen compiler configuration via a SystemML
configuration property. The default is 'auto', which is equivalent to
the previous behavior. However, now users can explicitly set the codegen
compiler, which is useful for programmatic APIs such as JMLC.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f45493e8
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f45493e8
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f45493e8
Branch: refs/heads/master
Commit: f45493e8aabb33ec1f628c016b3e68508dcf4ac8
Parents: 687e19c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Sep 11 14:00:36 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Sep 11 15:20:05 2017 -0700
----------------------------------------------------------------------
conf/SystemML-config.xml.template | 3 +++
src/main/java/org/apache/sysml/conf/DMLConfig.java | 11 +++++++----
.../org/apache/sysml/hops/codegen/SpoofCompiler.java | 9 ++++++++-
3 files changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/f45493e8/conf/SystemML-config.xml.template
----------------------------------------------------------------------
diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template
index 86bacda..454d0cc 100644
--- a/conf/SystemML-config.xml.template
+++ b/conf/SystemML-config.xml.template
@@ -60,6 +60,9 @@
<!-- enables operator fusion via code generation, experimental feature -->
<codegen.enabled>false</codegen.enabled>
+ <!-- set the codegen java compiler (auto, janino, javac) -->
+ <codegen.compiler>auto</codegen.compiler>
+
<!-- if codegen.enabled, enables source code caching of fused operators -->
<codegen.plancache>false</codegen.plancache>
http://git-wip-us.apache.org/repos/asf/systemml/blob/f45493e8/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index 84beea8..bfb3850 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -40,6 +40,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.codegen.SpoofCompiler.CompilerType;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.io.IOUtilFunctions;
@@ -74,6 +75,7 @@ public class DMLConfig
public static final String COMPRESSED_LINALG = "compressed.linalg";
public static final String NATIVE_BLAS = "native.blas";
public static final String CODEGEN = "codegen.enabled"; //boolean
+ public static final String CODEGEN_COMPILER = "codegen.compiler"; //see SpoofCompiler.CompilerType
public static final String CODEGEN_PLANCACHE = "codegen.plancache"; //boolean
public static final String CODEGEN_LITERALS = "codegen.literals"; //1..heuristic, 2..always
public static final String EXTRA_FINEGRAINED_STATS = "systemml.stats.finegrained"; //boolean
@@ -119,16 +121,16 @@ public class DMLConfig
_defaultVals.put(CP_PARALLEL_IO, "true" );
_defaultVals.put(COMPRESSED_LINALG, "false" );
_defaultVals.put(CODEGEN, "false" );
+ _defaultVals.put(CODEGEN_COMPILER, CompilerType.AUTO.name() );
_defaultVals.put(CODEGEN_PLANCACHE, "true" );
_defaultVals.put(CODEGEN_LITERALS, "1" );
_defaultVals.put(NATIVE_BLAS, "none" );
_defaultVals.put(EXTRA_FINEGRAINED_STATS,"false" );
- _defaultVals.put(STATS_MAX_WRAP_LEN,"30" );
+ _defaultVals.put(STATS_MAX_WRAP_LEN, "30" );
_defaultVals.put(EXTRA_GPU_STATS, "false" );
_defaultVals.put(EXTRA_DNN_STATS, "false" );
-
_defaultVals.put(GPU_MEMORY_UTILIZATION_FACTOR, "0.9" );
- _defaultVals.put(AVAILABLE_GPUS, "-1");
+ _defaultVals.put(AVAILABLE_GPUS, "-1");
}
public DMLConfig()
@@ -408,7 +410,8 @@ public class DMLConfig
NUM_REDUCERS, DEFAULT_BLOCK_SIZE,
YARN_APPMASTER, YARN_APPMASTERMEM, YARN_MAPREDUCEMEM,
CP_PARALLEL_OPS, CP_PARALLEL_IO, NATIVE_BLAS,
- COMPRESSED_LINALG, CODEGEN, CODEGEN_LITERALS, CODEGEN_PLANCACHE,
+ COMPRESSED_LINALG,
+ CODEGEN, CODEGEN_COMPILER, CODEGEN_PLANCACHE, CODEGEN_LITERALS,
EXTRA_GPU_STATS, EXTRA_DNN_STATS, EXTRA_FINEGRAINED_STATS, STATS_MAX_WRAP_LEN,
AVAILABLE_GPUS
};
http://git-wip-us.apache.org/repos/asf/systemml/blob/f45493e8/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
index ca3c233..b98342c 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
@@ -36,6 +36,8 @@ import org.apache.log4j.Logger;
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.codegen.cplan.CNode;
import org.apache.sysml.hops.codegen.cplan.CNodeCell;
import org.apache.sysml.hops.codegen.cplan.CNodeData;
@@ -114,6 +116,7 @@ public class SpoofCompiler
public static final PlanSelector PLAN_SEL_POLICY = PlanSelector.FUSE_COST_BASED_V2;
public enum CompilerType {
+ AUTO,
JAVAC,
JANINO,
}
@@ -484,7 +487,11 @@ public class SpoofCompiler
}
public static void setExecTypeSpecificJavaCompiler() {
- JAVA_COMPILER = OptimizerUtils.isSparkExecutionMode() ?
+ DMLConfig conf = ConfigurationManager.getDMLConfig();
+ String compiler = conf.getTextValue(DMLConfig.CODEGEN_COMPILER);
+ CompilerType type = CompilerType.valueOf(compiler.toUpperCase());
+ JAVA_COMPILER = (type != CompilerType.AUTO) ? type :
+ OptimizerUtils.isSparkExecutionMode() ?
CompilerType.JANINO : CompilerType.JAVAC;
}
[2/2] systemml git commit: [SYSTEMML-1901] Fix parfor support for
frame inputs and intermediates
Posted by mb...@apache.org.
[SYSTEMML-1901] Fix parfor support for frame inputs and intermediates
This patch fixes parfor optimizer and runtime issues regarding the
support of frame inputs and intermediates. Accordingly, this patch also
adds some related tests to avoid such issues in the future.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/75454819
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/75454819
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/75454819
Branch: refs/heads/master
Commit: 754548190a7ade052ab7218941bd2fa43b50b318
Parents: f45493e
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Sep 11 15:19:00 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Sep 11 15:20:06 2017 -0700
----------------------------------------------------------------------
.../controlprogram/ParForProgramBlock.java | 3 +-
.../parfor/opt/OptimizerRuleBased.java | 13 ++-
.../frame/ParforFrameIntermediateTest.java | 97 ++++++++++++++++++++
.../frame/ParforFrameIntermediates.dml | 29 ++++++
.../functions/frame/ZPackageSuite.java | 1 +
5 files changed, 135 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index b393cd1..4dda1b4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -1279,7 +1279,8 @@ public class ParForProgramBlock extends ForProgramBlock
}
break;
case MATRIX:
- //currently we do not create any unscoped matrix object outputs
+ case FRAME:
+ //currently we do not create any unscoped matrix or frame outputs
//because metadata (e.g., outputinfo) not known at this place.
break;
case UNKNOWN:
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index ec229d5..72b9566 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -456,26 +456,25 @@ public class OptimizerRuleBased extends Optimizer
{
boolean ret = false;
- if( !n.isLeaf() )
- {
+ if( !n.isLeaf() ) {
for( OptNode cn : n.getChilds() )
if( cn.getNodeType() != NodeType.FUNCCALL ) //prevent conflicts with aliases
ret |= rFindDataPartitioningCandidates( cn, cand, vars, thetaM );
}
else if( n.getNodeType()== NodeType.HOP
- && n.getParam(ParamType.OPSTRING).equals(IndexingOp.OPSTRING) )
+ && n.getParam(ParamType.OPSTRING).equals(IndexingOp.OPSTRING) )
{
Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
String inMatrix = h.getInput().get(0).getName();
- if( cand.containsKey(inMatrix) ) //Required Condition: partitioning applicable
+ if( cand.containsKey(inMatrix) && h.getDataType().isMatrix() ) //Required: partitionable
{
PartitionFormat dpf = cand.get(inMatrix);
double mnew = getNewRIXMemoryEstimate( n, inMatrix, dpf, vars );
//NOTE: for the moment, we do not partition according to the remote mem, because we can execute
- //it even without partitioning in CP. However, advanced optimizers should reason about this
+ //it even without partitioning in CP. However, advanced optimizers should reason about this
//double mold = h.getMemEstimate();
- if( n.getExecType() == getRemoteExecType() //Opt Condition: MR/Spark
- || h.getMemEstimate() > thetaM ) //Opt Condition: mem estimate > constraint to force partitioning
+ if( n.getExecType() == getRemoteExecType() //Opt Condition: MR/Spark
+ || h.getMemEstimate() > thetaM ) //Opt Condition: mem estimate > constraint to force partitioning
{
//NOTE: subsequent rewrites will still use the MR mem estimate
//(guarded by subsequent operations that have at least the memory req of one partition)
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java
new file mode 100644
index 0000000..c9ac763
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/ParforFrameIntermediateTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.io.FrameWriterFactory;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+public class ParforFrameIntermediateTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/frame/";
+ private final static String TEST_NAME = "ParforFrameIntermediates";
+ private final static String TEST_CLASS_DIR = TEST_DIR + ParforFrameIntermediateTest.class.getSimpleName() + "/";
+
+ private final static int rows = 1382;
+ private final static int cols = 5;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"F2"}));
+ }
+
+ @Test
+ public void testParforFrameIntermediatesCP() {
+ runParforFrameIntermediatesTest(ExecType.CP);
+ }
+
+ @Test
+ public void testParforFrameIntermediatesSpark() {
+ runParforFrameIntermediatesTest(ExecType.SPARK);
+ }
+
+ private void runParforFrameIntermediatesTest( ExecType et ) {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ){
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK
+ || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ //setup testcase
+ getAndLoadTestConfiguration(TEST_NAME);
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-explain", "-args", input("F")};
+
+ //generate input data and write as frame
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, 0.9, 8362);
+ FrameBlock fA = DataConverter.convertToFrameBlock(
+ DataConverter.convertToMatrixBlock(A));
+ FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo)
+ .writeFrameToHDFS(fA, input("F"), rows, cols);
+
+ //run test
+ runTest(true, false, null, -1);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/test/scripts/functions/frame/ParforFrameIntermediates.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/ParforFrameIntermediates.dml b/src/test/scripts/functions/frame/ParforFrameIntermediates.dml
new file mode 100644
index 0000000..1fee376
--- /dev/null
+++ b/src/test/scripts/functions/frame/ParforFrameIntermediates.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+F = read($1, data_type="frame", format="csv");
+
+parfor(i in 1:ncol(F)) {
+ Fi = F[, i];
+ Mi = as.matrix(Fi) + 7;
+ Fi2 = as.frame(Mi);
+ print(toString(Fi2[1:2,]));
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/75454819/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
index 45617ba..bbdb9d2 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
@@ -45,6 +45,7 @@ import org.junit.runners.Suite;
FrameScalarCastingTest.class,
FrameSchemaReadTest.class,
FrameSerializationTest.class,
+ ParforFrameIntermediateTest.class,
})