You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/11 20:18:10 UTC
[3/4] incubator-systemml git commit: [SYSTEMML-556] JMLC api
extension for input and output frames
[SYSTEMML-556] JMLC api extension for input and output frames
This patch adds input and output frames to the JMLC API. Frames are
specified by a mixed schema or implicit string schema and data is passed
as a two dimensional string array or directly as a FrameBlock.
Furthermore, this change also includes extended casting functionalities
between frames and string arrays.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5a2d888a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5a2d888a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5a2d888a
Branch: refs/heads/master
Commit: 5a2d888a77d9b5acb082c4a63149d96b5b5f5894
Parents: ccfe921
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Mar 10 21:25:31 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Mar 10 21:25:31 2016 -0800
----------------------------------------------------------------------
.../apache/sysml/api/jmlc/PreparedScript.java | 77 +++++++++++++++-----
.../apache/sysml/api/jmlc/ResultVariables.java | 31 +++++++-
.../controlprogram/caching/FrameObject.java | 4 +-
.../sysml/runtime/util/DataConverter.java | 58 +++++++++++++++
4 files changed, 148 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index 381f83c..3bc4fc3 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map.Entry;
import org.apache.sysml.api.DMLException;
@@ -31,6 +32,7 @@ import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
@@ -42,6 +44,7 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -140,27 +143,13 @@ public class PreparedScript
_vars.put(varname, scalar);
}
- /**
- *
- * @param varname
- * @param matrix
- * @throws DMLException
- */
- public void setMatrix(String varname, double[][] matrix)
- throws DMLException
- {
+ /** Binds a matrix object to a registered input variable. */
+ public void setMatrix(String varname, double[][] matrix) throws DMLException {
setMatrix(varname, matrix, false);
}
- /**
- *
- * @param varname
- * @param matrix
- * @throws DMLException
- */
- public void setMatrix(String varname, double[][] matrix, boolean reuse)
- throws DMLException
- {
+ /** Binds a matrix object to a registered input variable. */
+ public void setMatrix(String varname, double[][] matrix, boolean reuse) throws DMLException {
setMatrix(varname, DataConverter.convertToMatrixBlock(matrix), reuse);
}
@@ -197,6 +186,58 @@ public class PreparedScript
_inVarReuse.put(varname, mo);
}
}
+
+ /** Binds a frame object to a registered input variable. */
+ public void setFrame(String varname, String[][] frame) throws DMLException {
+ setFrame(varname, frame, false);
+ }
+
+ /** Binds a frame object to a registered input variable. */
+ public void setFrame(String varname, String[][] frame, List<ValueType> schema) throws DMLException {
+ setFrame(varname, frame, schema, false);
+ }
+
+ /** Binds a frame object to a registered input variable. */
+ public void setFrame(String varname, String[][] frame, boolean reuse) throws DMLException {
+ setFrame(varname, DataConverter.convertToFrameBlock(frame), reuse);
+ }
+
+ /** Binds a frame object to a registered input variable. */
+ public void setFrame(String varname, String[][] frame, List<ValueType> schema, boolean reuse) throws DMLException {
+ setFrame(varname, DataConverter.convertToFrameBlock(frame, schema), reuse);
+ }
+
+ /**
+ * Binds a frame object to a registered input variable.
+ * If reuse requested, then the input is guaranteed to be
+ * preserved over multiple <code>executeScript</code> calls.
+ *
+ * @param varname
+ * @param frame
+ * @param reuse
+ * @throws DMLException
+ */
+ public void setFrame(String varname, FrameBlock frame, boolean reuse)
+ throws DMLException
+ {
+ if( !_inVarnames.contains(varname) )
+ throw new DMLException("Unspecified input variable: "+varname);
+
+ DMLConfig conf = ConfigurationManager.getConfig();
+ String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
+
+ //create new frame object
+ String fname = scratch_space+"/"+varname;
+ FrameObject fo = new FrameObject(fname, frame);
+
+ //put create matrix wrapper into symbol table
+ _vars.put(varname, fo);
+ if( reuse ) {
+ //TODO buffer pool integration
+ //mo.enableCleanup(false); //prevent cleanup
+ _inVarReuse.put(varname, fo);
+ }
+ }
/**
* Remove all current values bound to input or output variables.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java b/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
index 87910df..a5bf74a 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java
@@ -23,8 +23,10 @@ import java.util.HashMap;
import java.util.Set;
import org.apache.sysml.api.DMLException;
+import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.instructions.cp.Data;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.DataConverter;
@@ -55,7 +57,7 @@ public class ResultVariables
/**
*
- * @param var
+ * @param varname
* @return
* @throws DMLException
*/
@@ -83,6 +85,33 @@ public class ResultVariables
/**
*
+ * @param varname
+ * @return
+ * @throws DMLException
+ */
+ public String[][] getFrame(String varname)
+ throws DMLException
+ {
+ if( !_out.containsKey(varname) )
+ throw new DMLException("Non-existing output variable: "+varname);
+
+ String[][] ret = null;
+ Data dat = _out.get(varname);
+
+ //basic checks for data type
+ if( !(dat instanceof FrameObject) )
+ throw new DMLException("Expected frame result '"+varname+"' not a frame.");
+
+ //convert output matrix to double array
+ FrameObject fo = (FrameObject)dat;
+ FrameBlock frame = fo.getData();
+ ret = DataConverter.convertToStringFrame(frame);
+
+ return ret;
+ }
+
+ /**
+ *
*
* @param ovar
* @param data
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 40c62c7..701b8c9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -19,8 +19,6 @@
package org.apache.sysml.runtime.controlprogram.caching;
-import java.util.List;
-
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -40,7 +38,7 @@ public class FrameObject extends CacheableData
super(DataType.FRAME, ValueType.UNKNOWN);
}
- public FrameObject(String fname, List<ValueType> schema, FrameBlock data) {
+ public FrameObject(String fname, FrameBlock data) {
this();
setFileName(fname);
setData(data);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index c75aabf..131bee6 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -642,6 +642,64 @@ public class DataConverter
}
/**
+ * Converts a frame block with arbitrary schema into a two dimensional
+ * string array.
+ *
+ * @param frame
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static String[][] convertToStringFrame(FrameBlock frame)
+ throws DMLRuntimeException
+ {
+ String[][] ret = new String[frame.getNumRows()][];
+ Iterator<String[]> iter = frame.getStringRowIterator();
+ for( int i=0; iter.hasNext(); i++ ) {
+ //deep copy output rows due to internal reuse
+ ret[i] = iter.next().clone();
+ }
+
+ return ret;
+ }
+
+ /**
+ * Converts a two dimensions string array into a frame block of
+ * value type string. If the given array is null or of length 0,
+ * we return an empty frame block.
+ *
+ * @param data
+ * @return
+ */
+ public static FrameBlock convertToFrameBlock(String[][] data) {
+ //check for empty frame block
+ if( data == null || data.length==0 )
+ return new FrameBlock();
+
+ //construct temporary schema
+ List<ValueType> schema = new ArrayList<ValueType>();
+ for( int j=0; j<data[0].length; j++ )
+ schema.add(ValueType.STRING);
+
+ //create frame block
+ return convertToFrameBlock(data, schema);
+ }
+
+ /**
+ *
+ * @param data
+ * @param schema
+ * @return
+ */
+ public static FrameBlock convertToFrameBlock(String[][] data, List<ValueType> schema) {
+ //check for empty frame block
+ if( data == null || data.length==0 )
+ return new FrameBlock();
+
+ //create frame block
+ return new FrameBlock(schema, data);
+ }
+
+ /**
* Converts a matrix block into a frame block of value type double.
*
* @param mb