You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/07/07 18:25:52 UTC
[1/2] systemml git commit: [SYSTEMML-1749] Fix result correctness
single-threaded csv frame reader
Repository: systemml
Updated Branches:
refs/heads/master 152eba1a7 -> 36effc54d
[SYSTEMML-1749] Fix result correctness single-threaded csv frame reader
This patch fixes the single-threaded csv frame reader, which produced
incorrect results if the used record reader returns multiple splits. The
underlying problem was that the contents of each read split were written
starting at position 0. We now correctly maintain the current row
position across splits.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/9e87f00a
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/9e87f00a
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/9e87f00a
Branch: refs/heads/master
Commit: 9e87f00ac94e6d368e968cf8ac183d1afa02d65b
Parents: 152eba1
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Jul 6 20:58:00 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Jul 7 11:25:30 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/runtime/io/FrameReaderTextCSV.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/9e87f00a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
index 9e10f2c..76da0de 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
@@ -113,11 +113,12 @@ public class FrameReaderTextCSV extends FrameReader
informat.configure(job);
InputSplit[] splits = informat.getSplits(job, 1);
splits = IOUtilFunctions.sortInputSplits(splits);
- for( int i=0; i<splits.length; i++ )
- readCSVFrameFromInputSplit(splits[i], informat, job, dest, schema, names, rlen, clen, 0, i==0);
+ for( int i=0, rpos=0; i<splits.length; i++ )
+ rpos = readCSVFrameFromInputSplit(splits[i], informat,
+ job, dest, schema, names, rlen, clen, rpos, i==0);
}
- protected final void readCSVFrameFromInputSplit( InputSplit split, InputFormat<LongWritable,Text> informat, JobConf job,
+ protected final int readCSVFrameFromInputSplit( InputSplit split, InputFormat<LongWritable,Text> informat, JobConf job,
FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen, int rl, boolean first)
throws IOException
{
@@ -184,6 +185,8 @@ public class FrameReaderTextCSV extends FrameReader
finally {
IOUtilFunctions.closeSilently(reader);
}
+
+ return row;
}
protected Pair<Integer,Integer> computeCSVSize( Path path, JobConf job, FileSystem fs)
[2/2] systemml git commit: [SYSTEMML-1750] Enable dynamic
recompilation in JMLC and single-node
Posted by mb...@apache.org.
[SYSTEMML-1750] Enable dynamic recompilation in JMLC and single-node
This patch exposed optional compiler configurations through JMLC
connections in order to enable dynamic recompilation and multi-threading
for significant performance improvements on moderately large data. For
example, on a MLogreg text classification scenario with 28,556 x
2,123,544 sparse inputs this patch improved the end-to-end performance
by more than 2x compared to a baseline without dynamic recompilation
(even though both used forced single-node execution).
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/36effc54
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/36effc54
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/36effc54
Branch: refs/heads/master
Commit: 36effc54d00507e22b1e17bbae657104d6b7fd8f
Parents: 9e87f00
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Jul 6 23:16:02 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Jul 7 11:25:32 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/api/jmlc/Connection.java | 23 +++++++-
.../org/apache/sysml/api/jmlc/JMLCProxy.java | 55 ++++++++++++++++++++
.../java/org/apache/sysml/hops/AggBinaryOp.java | 5 +-
.../java/org/apache/sysml/hops/AggUnaryOp.java | 5 +-
.../java/org/apache/sysml/hops/BinaryOp.java | 4 +-
.../org/apache/sysml/hops/ConvolutionOp.java | 4 +-
.../java/org/apache/sysml/hops/DataGenOp.java | 5 +-
src/main/java/org/apache/sysml/hops/DataOp.java | 6 +--
src/main/java/org/apache/sysml/hops/Hop.java | 31 +++++++++--
.../java/org/apache/sysml/hops/IndexingOp.java | 4 +-
.../org/apache/sysml/hops/LeftIndexingOp.java | 5 +-
.../java/org/apache/sysml/hops/MultipleOp.java | 26 ++++-----
.../sysml/hops/ParameterizedBuiltinOp.java | 4 +-
.../org/apache/sysml/hops/QuaternaryOp.java | 6 +--
.../java/org/apache/sysml/hops/ReorgOp.java | 6 +--
.../java/org/apache/sysml/hops/TernaryOp.java | 16 +++---
.../java/org/apache/sysml/hops/UnaryOp.java | 6 +--
.../apache/sysml/hops/recompile/Recompiler.java | 10 +++-
.../runtime/controlprogram/ProgramBlock.java | 4 --
19 files changed, 148 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 689ffc7..43fb3e0 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -128,6 +128,23 @@ public class Connection implements Closeable
}
/**
+ * Connection constructor, the starting point for any other JMLC API calls.
+ * This variant allows to enable a set of boolean compiler configurations.
+ *
+ * @param config one or many boolean compiler configurations to enable.
+ */
+ public Connection(CompilerConfig.ConfigType... configs) {
+ //basic constructor, which also constructs the compiler config
+ this();
+
+ //set optional compiler configurations in current config
+ CompilerConfig cconf = ConfigurationManager.getCompilerConfig();
+ for( ConfigType configType : configs )
+ cconf.set(configType, true);
+ ConfigurationManager.setLocalConfig(cconf);
+ }
+
+ /**
* Prepares (precompiles) a script and registers input and output variables.
*
* @param script string representing the DML or PyDML script
@@ -190,7 +207,9 @@ public class Connection implements Closeable
//final cleanup runtime prog
JMLCUtils.cleanupRuntimeProgram(rtprog, outputs);
- //System.out.println(Explain.explain(rtprog));
+ //activate thread-local proxy for dynamic recompilation
+ if( ConfigurationManager.isDynamicRecompilation() )
+ JMLCProxy.setActive(outputs);
}
catch(ParseException pe) {
// don't chain ParseException (for cleaner error output)
@@ -212,6 +231,8 @@ public class Connection implements Closeable
public void close() {
//clear thread-local dml / compiler configs
ConfigurationManager.clearLocalConfigs();
+ if( ConfigurationManager.isDynamicRecompilation() )
+ JMLCProxy.setActive(null);
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/api/jmlc/JMLCProxy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/JMLCProxy.java b/src/main/java/org/apache/sysml/api/jmlc/JMLCProxy.java
new file mode 100644
index 0000000..b17857b
--- /dev/null
+++ b/src/main/java/org/apache/sysml/api/jmlc/JMLCProxy.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.api.jmlc;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import org.apache.sysml.runtime.instructions.Instruction;
+
+/**
+ * This proxy provides thread-local access to output variables per connection
+ * in order to enable dynamic recompilation in JMLC.
+ */
+public class JMLCProxy
+{
+ private static ThreadLocal<HashSet<String>> _outputs = new ThreadLocal<HashSet<String>>() {
+ @Override
+ protected HashSet<String> initialValue() {
+ return null;
+ }
+ };
+
+ public static void setActive(String[] output) {
+ if( output != null )
+ _outputs.set(new HashSet<String>(Arrays.asList(output)));
+ else
+ _outputs.remove();
+ }
+
+ public static boolean isActive() {
+ return (_outputs.get() != null);
+ }
+
+ public static ArrayList<Instruction> performCleanupAfterRecompilation(ArrayList<Instruction> tmp) {
+ return JMLCUtils.cleanupRuntimeInstructions(tmp, _outputs.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
index c721efe..eb83549 100644
--- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
@@ -21,7 +21,6 @@ package org.apache.sysml.hops;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Binary;
@@ -436,9 +435,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
+ setRequiresRecompileIfNecessary();
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index eb469ab..e2f2a8e 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -20,7 +20,6 @@
package org.apache.sysml.hops;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
@@ -450,9 +449,7 @@ public class AggUnaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
+ setRequiresRecompileIfNecessary();
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/BinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/BinaryOp.java b/src/main/java/org/apache/sysml/hops/BinaryOp.java
index 36f573c..6175621 100644
--- a/src/main/java/org/apache/sysml/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/BinaryOp.java
@@ -1010,9 +1010,7 @@ public class BinaryOp extends Hop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
+ setRequiresRecompileIfNecessary();
//ensure cp exec type for single-node operations
if ( op == OpOp2.SOLVE ) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
index 8303e73..a3d8a81 100644
--- a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
+++ b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
@@ -19,7 +19,6 @@
package org.apache.sysml.hops;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.lops.ConvolutionTransform;
import org.apache.sysml.lops.ConvolutionTransform.OperationTypes;
@@ -335,8 +334,7 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop
_etype = !isEligibleForSpark() && _etype == REMOTE ? ExecType.CP : _etype;
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
+ setRequiresRecompileIfNecessary();
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataGenOp.java b/src/main/java/org/apache/sysml/hops/DataGenOp.java
index 3f29358..ce08dbc 100644
--- a/src/main/java/org/apache/sysml/hops/DataGenOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataGenOp.java
@@ -293,9 +293,8 @@ public class DataGenOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
+ setRequiresRecompileIfNecessary();
+
//always force string initialization into CP (not supported in MR)
//similarly, sample is currently not supported in MR either
if( _op == DataGenMethod.SINIT )
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/DataOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataOp.java b/src/main/java/org/apache/sysml/hops/DataOp.java
index 8653360..bcded04 100644
--- a/src/main/java/org/apache/sysml/hops/DataOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataOp.java
@@ -454,7 +454,7 @@ public class DataOp extends Hop
if( getDataType()==DataType.SCALAR || (getDataType()==DataType.FRAME && REMOTE==ExecType.MR) )
_etypeForced = ExecType.CP;
- if( _etypeForced != null )
+ if( _etypeForced != null )
{
_etype = _etypeForced;
}
@@ -478,9 +478,7 @@ public class DataOp extends Hop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
+ setRequiresRecompileIfNecessary();
}
else //READ
{
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 81c4cdc..8f8afde 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1472,17 +1472,38 @@ public abstract class Hop
/**
* Indicates if dynamic recompilation is required for this hop.
*
- * @return true if dynamic recompilcation required
+ * @return true if dynamic recompilation required
*/
- public boolean requiresRecompile()
- {
+ public boolean requiresRecompile() {
return _requiresRecompile;
}
- public void setRequiresRecompile()
- {
+ /**
+ * Marks the hop for dynamic recompilation.
+ */
+ public void setRequiresRecompile() {
_requiresRecompile = true;
}
+
+ /**
+ * Marks the hop for dynamic recompilation, if dynamic recompilation is
+ * enabled and one of the two basic scenarios apply:
+ * <ul>
+ * <li> The hop has unknown dimensions or sparsity and is scheduled for
+ * remote execution, in which case the latency for distributed jobs easily
+ * covers any recompilation overheads. </li>
+ * <li> The hop has unknown dimensions and is scheduled for local execution
+ * due to forced single node execution type. </li>
+ * <ul> <p>
+ */
+ protected void setRequiresRecompileIfNecessary() {
+ ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
+ boolean caseRemote = (!dimsKnown(true) && _etype == REMOTE);
+ boolean caseLocal = (!dimsKnown() && _etypeForced == ExecType.CP);
+
+ if( ConfigurationManager.isDynamicRecompilation() && (caseRemote || caseLocal) )
+ setRequiresRecompile();
+ }
/**
* Update the output size information for this hop.
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/IndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java b/src/main/java/org/apache/sysml/hops/IndexingOp.java
index e446552..5a27ed6 100644
--- a/src/main/java/org/apache/sysml/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java
@@ -19,7 +19,6 @@
package org.apache.sysml.hops;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
@@ -376,8 +375,7 @@ public class IndexingOp extends Hop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
+ setRequiresRecompileIfNecessary();
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
index c6769d4..601cb6f 100644
--- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
@@ -389,9 +389,8 @@ public class LeftIndexingOp extends Hop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
+ setRequiresRecompileIfNecessary();
+
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/MultipleOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/MultipleOp.java b/src/main/java/org/apache/sysml/hops/MultipleOp.java
index 688204f..5fb6b29 100644
--- a/src/main/java/org/apache/sysml/hops/MultipleOp.java
+++ b/src/main/java/org/apache/sysml/hops/MultipleOp.java
@@ -35,7 +35,7 @@ import org.apache.sysml.parser.Expression.ValueType;
*
*/
public class MultipleOp extends Hop {
- protected MultiInputOp multipleOperandOperation = null;
+ protected MultiInputOp _op = null;
protected MultipleOp() {
}
@@ -60,15 +60,12 @@ public class MultipleOp extends Hop {
public MultipleOp(String name, DataType dataType, ValueType valueType,
MultiInputOp multipleOperandOperation, Hop... inputs) throws HopsException {
super(name, dataType, valueType);
- this.multipleOperandOperation = multipleOperandOperation;
+ _op = multipleOperandOperation;
for (int i = 0; i < inputs.length; i++) {
getInput().add(i, inputs[i]);
inputs[i].getParent().add(this);
}
-
- // compute unknown dims and nnz
- refreshSizeInformation();
}
/** MultipleOp may have any number of inputs. */
@@ -76,12 +73,12 @@ public class MultipleOp extends Hop {
public void checkArity() throws HopsException {}
public MultiInputOp getOp() {
- return multipleOperandOperation;
+ return _op;
}
@Override
public String getOpString() {
- return "m(" + multipleOperandOperation.toString().toLowerCase() + ")";
+ return "m(" + _op.name().toLowerCase() + ")";
}
/**
@@ -102,10 +99,10 @@ public class MultipleOp extends Hop {
inLops[i] = inLop;
}
- MultipleCP.OperationType opType = MultipleOperandOperationHopTypeToLopType.get(multipleOperandOperation);
+ MultipleCP.OperationType opType = MultipleOperandOperationHopTypeToLopType.get(_op);
if (opType == null) {
- throw new HopsException("Unknown MultipleCP Lop operation type for MultipleOperandOperation Hop type '"
- + multipleOperandOperation + "'");
+ throw new HopsException("Unknown MultipleCP Lop operation type for "
+ + "MultipleOperandOperation Hop type '" + _op + "'");
}
MultipleCP multipleCPLop = new MultipleCP(opType, getDataType(), getValueType(), inLops);
@@ -130,12 +127,11 @@ public class MultipleOp extends Hop {
@Override
public boolean allowsAllExecTypes() {
- return false; // true?
+ return false;
}
@Override
protected ExecType optFindExecType() throws HopsException {
- checkAndSetForcedPlatform(); // ?
return ExecType.CP;
}
@@ -152,7 +148,7 @@ public class MultipleOp extends Hop {
multipleOp.clone(this, false);
// copy specific attributes
- multipleOp.multipleOperandOperation = multipleOperandOperation;
+ multipleOp._op = _op;
return multipleOp;
}
@@ -162,14 +158,14 @@ public class MultipleOp extends Hop {
if (!(that instanceof MultipleOp))
return false;
- if (multipleOperandOperation == MultiInputOp.PRINTF) {
+ if (_op == MultiInputOp.PRINTF) {
return false;
}
// if add new multiple operand types in addition to PRINTF,
// probably need to modify this.
MultipleOp mo = (MultipleOp) that;
- return (multipleOperandOperation == mo.multipleOperandOperation);
+ return (_op == mo._op);
}
@Override
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index 88ec360..7d6fe2a 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -22,7 +22,6 @@ package org.apache.sysml.hops;
import java.util.HashMap;
import java.util.Map.Entry;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
@@ -1103,8 +1102,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
+ setRequiresRecompileIfNecessary();
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
index e3dfa54..6517de6 100644
--- a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
@@ -19,7 +19,6 @@
package org.apache.sysml.hops;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.DataPartition;
@@ -1490,9 +1489,8 @@ public class QuaternaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
+ setRequiresRecompileIfNecessary();
+
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/ReorgOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ReorgOp.java b/src/main/java/org/apache/sysml/hops/ReorgOp.java
index 8ed308b..20cd68d 100644
--- a/src/main/java/org/apache/sysml/hops/ReorgOp.java
+++ b/src/main/java/org/apache/sysml/hops/ReorgOp.java
@@ -22,7 +22,6 @@ package org.apache.sysml.hops;
import java.util.ArrayList;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
@@ -540,9 +539,8 @@ public class ReorgOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
+ setRequiresRecompileIfNecessary();
+
return _etype;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/TernaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/TernaryOp.java b/src/main/java/org/apache/sysml/hops/TernaryOp.java
index 458a346..5a12dea 100644
--- a/src/main/java/org/apache/sysml/hops/TernaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/TernaryOp.java
@@ -849,12 +849,12 @@ public class TernaryOp extends Hop
}
//mark for recompile (forever)
- // Necessary condition for recompilation is unknown dimensions.
- // When execType=CP, it is marked for recompilation only when additional
- // dimension inputs are provided (and those values are unknown at initial compile time).
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) ) {
- if ( _etype==REMOTE || (_etype == ExecType.CP && _dimInputsPresent))
- setRequiresRecompile();
+ // additional condition: when execType=CP and additional dimension inputs
+ // are provided (and those values are unknown at initial compile time).
+ setRequiresRecompileIfNecessary();
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true)
+ && _etype == ExecType.CP && _dimInputsPresent) {
+ setRequiresRecompile();
}
return _etype;
@@ -1079,10 +1079,8 @@ public class TernaryOp extends Hop
}
}
}
- catch(Exception ex)
- {
+ catch(Exception ex) {
throw new RuntimeException(ex);
- //ret = false;
}
return ret;
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 176d131..d90fcdf 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -21,7 +21,6 @@ package org.apache.sysml.hops;
import java.util.ArrayList;
-import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Aggregate.OperationTypes;
@@ -667,9 +666,8 @@ public class UnaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
+ setRequiresRecompileIfNecessary();
+
//ensure cp exec type for single-node operations
if( _op == OpOp1.PRINT || _op == OpOp1.STOP
|| _op == OpOp1.INVERSE || _op == OpOp1.EIGEN || _op == OpOp1.CHOLESKY )
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 15376ae..04d521b7 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.wink.json4j.JSONObject;
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.jmlc.JMLCProxy;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.conf.CompilerConfig.ConfigType;
@@ -105,6 +106,7 @@ import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.utils.Explain;
import org.apache.sysml.utils.Explain.ExplainType;
import org.apache.sysml.utils.JSONHelper;
+import org.apache.sysml.utils.MLContextProxy;
/**
* Dynamic recompilation of hop dags to runtime instructions, which includes the
@@ -235,13 +237,19 @@ public class Recompiler
}
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
+ newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
if( tid != 0 ) //only in parfor context
newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
+ // remove writes if called through mlcontext or jmlc
+ if( MLContextProxy.isActive() )
+ newInst = MLContextProxy.performCleanupAfterRecompilation(newInst);
+ else if( JMLCProxy.isActive() )
+ newInst = JMLCProxy.performCleanupAfterRecompilation(newInst);
+
// explain recompiled hops / instructions
if( DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS ){
LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+"):\n" +
http://git-wip-us.apache.org/repos/asf/systemml/blob/36effc54/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index 5ae5d80..209c566 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -46,7 +46,6 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.utils.MLContextProxy;
import org.apache.sysml.utils.Statistics;
import org.apache.sysml.yarn.DMLAppMasterUtils;
@@ -147,9 +146,6 @@ public class ProgramBlock
{
tmp = Recompiler.recompileHopsDag(
_sb, _sb.get_hops(), ec.getVariables(), null, false, true, _tid);
-
- if( MLContextProxy.isActive() )
- tmp = MLContextProxy.performCleanupAfterRecompilation(tmp);
}
if( DMLScript.STATISTICS ){
long t1 = System.nanoTime();