You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/04/03 23:56:26 UTC
incubator-systemml git commit: [SYSTEMML-612] New cp frame right
indexing and append operations, tests
Repository: incubator-systemml
Updated Branches:
refs/heads/master 4d5e66054 -> 36612f987
[SYSTEMML-612] New cp frame right indexing and append operations, tests
This patch introduces cp right indexing and append operations over
frames. Due to the missing frame reader/writer integration these
operations are currently only usable through JMLC but apply without
change to CP in general. Furthermore, this change also cleans up the
existing indexing and append operations over matrices and scalars
(separate instructions, convenience block operations).
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/36612f98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/36612f98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/36612f98
Branch: refs/heads/master
Commit: 36612f98703f04b274559e6cf17280c9fb6618e2
Parents: 4d5e660
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Apr 3 01:33:14 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Apr 3 14:55:56 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/api/jmlc/PreparedScript.java | 4 +-
.../java/org/apache/sysml/hops/BinaryOp.java | 5 +-
.../sysml/parser/BuiltinFunctionExpression.java | 17 +-
.../controlprogram/caching/FrameObject.java | 48 ++++-
.../context/ExecutionContext.java | 7 +-
.../instructions/CPInstructionParser.java | 4 +-
.../instructions/cp/AppendCPInstruction.java | 93 ++--------
.../cp/FrameAppendCPInstruction.java | 60 +++++++
.../cp/FrameIndexingCPInstruction.java | 62 +++++++
.../instructions/cp/IndexingCPInstruction.java | 132 ++++++++++++++
.../cp/MatrixAppendCPInstruction.java | 60 +++++++
.../cp/MatrixIndexingCPInstruction.java | 112 ++----------
.../cp/ScalarAppendCPInstruction.java | 55 ++++++
.../cpfile/MatrixIndexingCPFileInstruction.java | 23 +--
.../sysml/runtime/matrix/data/FrameBlock.java | 16 ++
.../sysml/runtime/matrix/data/MatrixBlock.java | 30 ++++
.../apache/sysml/runtime/util/IndexRange.java | 31 ++--
.../functions/jmlc/FrameIndexingAppendTest.java | 173 +++++++++++++++++++
src/test/scripts/functions/jmlc/transform4.dml | 46 +++++
19 files changed, 767 insertions(+), 211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index d68199e..f25d19d 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -235,7 +235,9 @@ public class PreparedScript
String scratch_space = ConfigurationManager.getScratchSpace();
//create new frame object
- FrameObject fo = new FrameObject(scratch_space+"/"+varname);
+ MatrixCharacteristics mc = new MatrixCharacteristics(frame.getNumRows(), frame.getNumColumns(), -1, -1);
+ MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryCellOutputInfo, InputInfo.BinaryCellInputInfo);
+ FrameObject fo = new FrameObject(scratch_space+"/"+varname, meta);
fo.acquireModify(frame);
fo.release();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/hops/BinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/BinaryOp.java b/src/main/java/org/apache/sysml/hops/BinaryOp.java
index e434197..94de0e7 100644
--- a/src/main/java/org/apache/sysml/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/BinaryOp.java
@@ -503,14 +503,15 @@ public class BinaryOp extends Hop
//sanity check for input data types
if( !((dt1==DataType.MATRIX && dt2==DataType.MATRIX)
+ ||(dt1==DataType.FRAME && dt2==DataType.FRAME)
||(dt1==DataType.SCALAR && dt2==DataType.SCALAR
&& vt1==ValueType.STRING && vt2==ValueType.STRING )) )
{
- throw new HopsException("Append can only apply to two matrices or two scalar strings!");
+ throw new HopsException("Append can only apply to two matrices, two frames, or two scalar strings!");
}
Lop append = null;
- if( dt1==DataType.MATRIX && dt2==DataType.MATRIX )
+ if( dt1==DataType.MATRIX || dt1==DataType.FRAME )
{
long rlen = cbind ? getInput().get(0).getDim1() : (getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ?
getInput().get(0).getDim1()+getInput().get(1).getDim1() : -1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
index 9273590..886aa0a 100644
--- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
@@ -409,8 +409,8 @@ public class BuiltinFunctionExpression extends DataIdentifier
}
//matrix append (rbind/cbind)
else {
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
+ checkMatrixFrameParam(getFirstExpr());
+ checkMatrixFrameParam(getSecondExpr());
}
output.setDataType(id.getDataType());
@@ -1181,6 +1181,19 @@ public class BuiltinFunctionExpression extends DataIdentifier
* @param e
* @throws LanguageException
*/
+ protected void checkMatrixFrameParam(Expression e) //always unconditional
+ throws LanguageException
+ {
+ if (e.getOutput().getDataType() != DataType.MATRIX && e.getOutput().getDataType() != DataType.FRAME) {
+ raiseValidateError("Expecting matrix or frame parameter for function "+ this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
+ }
+ }
+
+ /**
+ *
+ * @param e
+ * @throws LanguageException
+ */
private void checkScalarParam(Expression e) //always unconditional
throws LanguageException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 375198d..0055a51 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -22,6 +22,9 @@ package org.apache.sysml.runtime.controlprogram.caching;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -29,6 +32,9 @@ public class FrameObject extends CacheableData<FrameBlock>
{
private static final long serialVersionUID = 1755082174281927785L;
+ /** Holds meta data on frame characteristics (required for nrow etc)*/
+ private MetaData _metaData = null;
+
/**
*
*/
@@ -46,6 +52,17 @@ public class FrameObject extends CacheableData<FrameBlock>
}
/**
+ *
+ * @param fname
+ * @param meta
+ */
+ public FrameObject(String fname, MetaData meta) {
+ this();
+ setFileName(fname);
+ setMetaData(meta);
+ }
+
+ /**
* Copy constructor that copies meta data but NO data.
*
* @param fo
@@ -54,6 +71,31 @@ public class FrameObject extends CacheableData<FrameBlock>
super(fo);
}
+ @Override
+ public void setMetaData(MetaData md) {
+ _metaData = md;
+ }
+
+ @Override
+ public MetaData getMetaData() {
+ return _metaData;
+ }
+
+ @Override
+ public void removeMetaData() {
+ _metaData = null;
+ }
+
+ public void refreshMetaData()
+ throws CacheException
+ {
+ if ( _data == null || _metaData ==null ) //refresh only for existing data
+ throw new CacheException("Cannot refresh meta data because there is no data or meta data. ");
+
+ MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
+ mc.setDimension( _data.getNumRows(),_data.getNumColumns() );
+ }
+
////////////////////////////////////
// high-level cache API (pseudo-integrated)
@@ -81,7 +123,11 @@ public class FrameObject extends CacheableData<FrameBlock>
public FrameBlock acquireModify(FrameBlock newData)
throws CacheException
{
- return _data = newData;
+ //set data and update meta data
+ _data = newData;
+ refreshMetaData();
+
+ return _data;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 05d23a7..f56a870 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -45,9 +45,12 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.util.MapReduceTool;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -310,7 +313,9 @@ public class ExecutionContext
{
//TODO: fix createvar instructions generation for frame handles
//FrameObject fo = (FrameObject) getVariable(varName);
- FrameObject fo = new FrameObject(varName);
+ MatrixCharacteristics mc = new MatrixCharacteristics(-1, -1, -1, -1);
+ MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryCellOutputInfo, InputInfo.BinaryCellInputInfo);
+ FrameObject fo = new FrameObject(varName, meta);
fo.acquireModify(outputData);
fo.release();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index aae821f..8ffc4e7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -42,9 +42,9 @@ import org.apache.sysml.runtime.instructions.cp.DataGenCPInstruction;
import org.apache.sysml.runtime.instructions.cp.DataPartitionCPInstruction;
import org.apache.sysml.runtime.instructions.cp.FileCPInstruction;
import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.IndexingCPInstruction;
import org.apache.sysml.runtime.instructions.cp.MMChainCPInstruction;
import org.apache.sysml.runtime.instructions.cp.MMTSJCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.MatrixIndexingCPInstruction;
import org.apache.sysml.runtime.instructions.cp.MatrixReshapeCPInstruction;
import org.apache.sysml.runtime.instructions.cp.MultiReturnBuiltinCPInstruction;
import org.apache.sysml.runtime.instructions.cp.PMMJCPInstruction;
@@ -359,7 +359,7 @@ public class CPInstructionParser extends InstructionParser
case MatrixIndexing:
execType = ExecType.valueOf( str.split(Instruction.OPERAND_DELIM)[0] );
if( execType == ExecType.CP )
- return MatrixIndexingCPInstruction.parseInstruction(str);
+ return IndexingCPInstruction.parseInstruction(str);
else //exectype CP_FILE
return MatrixIndexingCPFileInstruction.parseInstruction(str);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
index 41cb68a..15482fd 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AppendCPInstruction.java
@@ -21,16 +21,13 @@ package org.apache.sysml.runtime.instructions.cp;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.DMLUnsupportedOperationException;
-import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.functionobjects.OffsetColumnIndex;
import org.apache.sysml.runtime.instructions.InstructionUtils;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
-public class AppendCPInstruction extends BinaryCPInstruction
+public abstract class AppendCPInstruction extends BinaryCPInstruction
{
public enum AppendType{
CBIND,
@@ -39,7 +36,7 @@ public class AppendCPInstruction extends BinaryCPInstruction
}
//type (matrix cbind / scalar string concatenation)
- private AppendType _type;
+ protected AppendType _type;
public AppendCPInstruction(Operator op, CPOperand in1, CPOperand in2, CPOperand in3, CPOperand out, AppendType type, String opcode, String istr)
{
@@ -49,6 +46,12 @@ public class AppendCPInstruction extends BinaryCPInstruction
_type = type;
}
+ /**
+ *
+ * @param str
+ * @return
+ * @throws DMLRuntimeException
+ */
public static AppendCPInstruction parseInstruction ( String str )
throws DMLRuntimeException
{
@@ -62,80 +65,18 @@ public class AppendCPInstruction extends BinaryCPInstruction
CPOperand out = new CPOperand(parts[4]);
boolean cbind = Boolean.parseBoolean(parts[5]);
- AppendType type = (in1.getDataType()!=DataType.MATRIX) ? AppendType.STRING :
- cbind ? AppendType.CBIND : AppendType.RBIND;
+ AppendType type = (in1.getDataType()!=DataType.MATRIX && in1.getDataType()!=DataType.FRAME) ?
+ AppendType.STRING : cbind ? AppendType.CBIND : AppendType.RBIND;
if(!opcode.equalsIgnoreCase("append"))
throw new DMLRuntimeException("Unknown opcode while parsing a AppendCPInstruction: " + str);
- return new AppendCPInstruction(
- new ReorgOperator(OffsetColumnIndex.getOffsetColumnIndexFnObject(-1)),
- in1, in2, in3, out, type, opcode, str);
- }
-
- @Override
- public void processInstruction(ExecutionContext ec)
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- if( _type == AppendType.CBIND )
- {
- //get inputs
- MatrixBlock matBlock1 = ec.getMatrixInput(input1.getName());
- MatrixBlock matBlock2 = ec.getMatrixInput(input2.getName());
-
- //check input dimensions
- if(matBlock1.getNumRows() != matBlock2.getNumRows()) {
- throw new DMLRuntimeException("Append-cbind is not possible for input matrices "
- + input1.getName() + " and " + input2.getName()
- + " with different number of rows: "+matBlock1.getNumRows()+" vs "+matBlock2.getNumRows());
- }
-
- //execute append operations (append both inputs to initially empty output)
- MatrixBlock ret = matBlock1.appendOperations(matBlock2, new MatrixBlock(), true);
-
- //set output and release inputs
- ec.setMatrixOutput(output.getName(), ret);
- ec.releaseMatrixInput(input1.getName());
- ec.releaseMatrixInput(input2.getName());
- }
- else if( _type == AppendType.RBIND )
- {
- //get inputs
- MatrixBlock matBlock1 = ec.getMatrixInput(input1.getName());
- MatrixBlock matBlock2 = ec.getMatrixInput(input2.getName());
-
- //check input dimensions
- if(matBlock1.getNumColumns() != matBlock2.getNumColumns()) {
- throw new DMLRuntimeException("Append-rbind is not possible for input matrices "
- + input1.getName() + " and " + input2.getName()
- + " with different number of columns: "+matBlock1.getNumColumns()+" vs "+matBlock2.getNumColumns());
- }
-
- //execute append operations (append both inputs to initially empty output)
- MatrixBlock ret = matBlock1.appendOperations(matBlock2, new MatrixBlock(), false);
-
- //set output and release inputs
- ec.setMatrixOutput(output.getName(), ret);
- ec.releaseMatrixInput(input1.getName());
- ec.releaseMatrixInput(input2.getName());
- }
- else //STRING
- {
- //get input strings (vars or literals)
- ScalarObject so1 = ec.getScalarInput( input1.getName(), input1.getValueType(), input1.isLiteral() );
- ScalarObject so2 = ec.getScalarInput( input2.getName(), input2.getValueType(), input2.isLiteral() );
-
- //pre-checks
- String val1 = so1.getStringValue();
- String val2 = so2.getStringValue();
- StringObject.checkMaxStringLength( val1.length()+val2.length() );
-
- //core execution
- String outString = val1 + "\n" + val2;
- ScalarObject sores = new StringObject(outString);
-
- //set output
- ec.setScalarOutput(output.getName(), sores);
- }
+ Operator op = new ReorgOperator(OffsetColumnIndex.getOffsetColumnIndexFnObject(-1));
+ if( type == AppendType.STRING )
+ return new ScalarAppendCPInstruction(op, in1, in2, in3, out, type, opcode, str);
+ else if( in1.getDataType()==DataType.MATRIX )
+ return new MatrixAppendCPInstruction(op, in1, in2, in3, out, type, opcode, str);
+ else //DataType.FRAME
+ return new FrameAppendCPInstruction(op, in1, in2, in3, out, type, opcode, str);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameAppendCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameAppendCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameAppendCPInstruction.java
new file mode 100644
index 0000000..de012b5
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameAppendCPInstruction.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+
+public final class FrameAppendCPInstruction extends AppendCPInstruction
+{
+ public FrameAppendCPInstruction(Operator op, CPOperand in1, CPOperand in2, CPOperand in3, CPOperand out, AppendType type, String opcode, String istr) {
+ super(op, in1, in2, in3, out, type, opcode, istr);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec)
+ throws DMLUnsupportedOperationException, DMLRuntimeException
+ {
+ //get inputs
+ FrameBlock fin1 = ec.getFrameInput(input1.getName());
+ FrameBlock fin2 = ec.getFrameInput(input2.getName());
+
+ //check input dimensions
+ if( _type == AppendType.CBIND && fin1.getNumRows() != fin2.getNumRows() ) {
+ throw new DMLRuntimeException("Append-cbind is not possible for input matrices " + input1.getName() + " and " + input2.getName()
+ + " with different number of rows: "+fin1.getNumRows()+" vs "+fin2.getNumRows());
+ }
+ else if( _type == AppendType.RBIND && fin1.getNumColumns() != fin2.getNumColumns()) {
+ throw new DMLRuntimeException("Append-rbind is not possible for input matrices " + input1.getName() + " and " + input2.getName()
+ + " with different number of columns: "+fin1.getNumColumns()+" vs "+fin2.getNumColumns());
+ }
+
+ //execute append operations (append both inputs to initially empty output)
+ FrameBlock ret = fin1.appendOperations(fin2, new FrameBlock(), _type==AppendType.CBIND);
+
+ //set output and release inputs
+ ec.setFrameOutput(output.getName(), ret);
+ ec.releaseFrameInput(input1.getName());
+ ec.releaseFrameInput(input2.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java
new file mode 100644
index 0000000..6c2ed64
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.util.IndexRange;
+
+public final class FrameIndexingCPInstruction extends IndexingCPInstruction
+{
+ public FrameIndexingCPInstruction(Operator op, CPOperand in, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr){
+ super(op, in, rl, ru, cl, cu, out, opcode, istr);
+ }
+
+ public FrameIndexingCPInstruction(Operator op, CPOperand lhsInput, CPOperand rhsInput, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr){
+ super(op, lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, istr);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec)
+ throws DMLUnsupportedOperationException, DMLRuntimeException
+ {
+ String opcode = getOpcode();
+ IndexRange ixrange = getIndexRange(ec);
+
+ //right indexing
+ if( opcode.equalsIgnoreCase("rangeReIndex") )
+ {
+ //execute right indexing operation
+ FrameBlock in = ec.getFrameInput(input1.getName());
+ FrameBlock out = in.sliceOperations(ixrange, new FrameBlock());
+
+ //unpin rhs input
+ ec.releaseFrameInput(input1.getName());
+
+ //unpin output
+ ec.setFrameOutput(output.getName(), out);
+ }
+ else
+ throw new DMLRuntimeException("Invalid opcode (" + opcode +") encountered in FrameIndexingCPInstruction.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
new file mode 100644
index 0000000..402f0d5
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+import org.apache.sysml.runtime.util.IndexRange;
+
+public abstract class IndexingCPInstruction extends UnaryCPInstruction
+{
+ protected CPOperand rowLower, rowUpper, colLower, colUpper;
+
+ public IndexingCPInstruction(Operator op, CPOperand in, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr){
+ super(op, in, out, opcode, istr);
+ rowLower = rl;
+ rowUpper = ru;
+ colLower = cl;
+ colUpper = cu;
+ }
+
+ public IndexingCPInstruction(Operator op, CPOperand lhsInput, CPOperand rhsInput, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr){
+ super(op, lhsInput, rhsInput, out, opcode, istr);
+ rowLower = rl;
+ rowUpper = ru;
+ colLower = cl;
+ colUpper = cu;
+ }
+
+ /**
+ *
+ * @param ec
+ * @return
+ * @throws DMLRuntimeException
+ */
+ protected IndexRange getIndexRange(ExecutionContext ec) throws DMLRuntimeException {
+ return new IndexRange( //rl, ru, cl, ru
+ (int)(ec.getScalarInput(rowLower.getName(), rowLower.getValueType(), rowLower.isLiteral()).getLongValue()-1),
+ (int)(ec.getScalarInput(rowUpper.getName(), rowUpper.getValueType(), rowUpper.isLiteral()).getLongValue()-1),
+ (int)(ec.getScalarInput(colLower.getName(), colLower.getValueType(), colLower.isLiteral()).getLongValue()-1),
+ (int)(ec.getScalarInput(colUpper.getName(), colUpper.getValueType(), colUpper.isLiteral()).getLongValue()-1));
+ }
+
+ /**
+ *
+ * @param str
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static IndexingCPInstruction parseInstruction ( String str )
+ throws DMLRuntimeException
+ {
+ String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
+ String opcode = parts[0];
+
+ if ( opcode.equalsIgnoreCase("rangeReIndex") ) {
+ if ( parts.length == 7 ) {
+ // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
+ CPOperand in, rl, ru, cl, cu, out;
+ in = new CPOperand();
+ rl = new CPOperand();
+ ru = new CPOperand();
+ cl = new CPOperand();
+ cu = new CPOperand();
+ out = new CPOperand();
+ in.split(parts[1]);
+ rl.split(parts[2]);
+ ru.split(parts[3]);
+ cl.split(parts[4]);
+ cu.split(parts[5]);
+ out.split(parts[6]);
+ if( in.getDataType()==DataType.MATRIX )
+ return new MatrixIndexingCPInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str);
+ else //DataType.FRAME
+ return new FrameIndexingCPInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str);
+ }
+ else {
+ throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
+ }
+ }
+ else if ( opcode.equalsIgnoreCase("leftIndex")) {
+ if ( parts.length == 8 ) {
+ // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7
+ CPOperand lhsInput, rhsInput, rl, ru, cl, cu, out;
+ lhsInput = new CPOperand();
+ rhsInput = new CPOperand();
+ rl = new CPOperand();
+ ru = new CPOperand();
+ cl = new CPOperand();
+ cu = new CPOperand();
+ out = new CPOperand();
+ lhsInput.split(parts[1]);
+ rhsInput.split(parts[2]);
+ rl.split(parts[3]);
+ ru.split(parts[4]);
+ cl.split(parts[5]);
+ cu.split(parts[6]);
+ out.split(parts[7]);
+ if( lhsInput.getDataType()==DataType.MATRIX )
+ return new MatrixIndexingCPInstruction(new SimpleOperator(null), lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, str);
+ else //DataType.FRAME
+ return new FrameIndexingCPInstruction(new SimpleOperator(null), lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, str);
+ }
+ else {
+ throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
+ }
+ }
+ else {
+ throw new DMLRuntimeException("Unknown opcode while parsing a MatrixIndexingCPInstruction: " + str);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixAppendCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixAppendCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixAppendCPInstruction.java
new file mode 100644
index 0000000..db34958
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixAppendCPInstruction.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+
+public final class MatrixAppendCPInstruction extends AppendCPInstruction
+{
+ public MatrixAppendCPInstruction(Operator op, CPOperand in1, CPOperand in2, CPOperand in3, CPOperand out, AppendType type, String opcode, String istr) {
+ super(op, in1, in2, in3, out, type, opcode, istr);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec)
+ throws DMLUnsupportedOperationException, DMLRuntimeException
+ {
+ //get inputs
+ MatrixBlock matBlock1 = ec.getMatrixInput(input1.getName());
+ MatrixBlock matBlock2 = ec.getMatrixInput(input2.getName());
+
+ //check input dimensions
+ if( _type == AppendType.CBIND && matBlock1.getNumRows() != matBlock2.getNumRows() ) {
+ throw new DMLRuntimeException("Append-cbind is not possible for input matrices " + input1.getName() + " and " + input2.getName()
+ + " with different number of rows: "+matBlock1.getNumRows()+" vs "+matBlock2.getNumRows());
+ }
+ else if( _type == AppendType.RBIND && matBlock1.getNumColumns() != matBlock2.getNumColumns()) {
+ throw new DMLRuntimeException("Append-rbind is not possible for input matrices " + input1.getName() + " and " + input2.getName()
+ + " with different number of columns: "+matBlock1.getNumColumns()+" vs "+matBlock2.getNumColumns());
+ }
+
+ //execute append operations (append both inputs to initially empty output)
+ MatrixBlock ret = matBlock1.appendOperations(matBlock2, new MatrixBlock(), _type==AppendType.CBIND);
+
+ //set output and release inputs
+ ec.setMatrixOutput(output.getName(), ret);
+ ec.releaseMatrixInput(input1.getName());
+ ec.releaseMatrixInput(input2.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
index bfd2b0c..91d2c39 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
@@ -26,102 +26,19 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.operators.Operator;
-import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
import org.apache.sysml.runtime.util.IndexRange;
import org.apache.sysml.utils.Statistics;
-public class MatrixIndexingCPInstruction extends UnaryCPInstruction
-{
-
- /*
- * This class implements the matrix indexing functionality inside CP.
- * Example instructions:
- * rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
- * input=mVar1, output=mVar6,
- * bounds = (Var2,Var3,Var4,Var5)
- * rowindex_lower: Var2, rowindex_upper: Var3
- * colindex_lower: Var4, colindex_upper: Var5
- * leftIndex:mVar1:mVar2:Var3:Var4:Var5:Var6:mVar7
- * triggered by "mVar1[Var3:Var4, Var5:Var6] = mVar2"
- * the result is stored in mVar7
- *
- */
- protected CPOperand rowLower, rowUpper, colLower, colUpper;
-
+public final class MatrixIndexingCPInstruction extends IndexingCPInstruction
+{
public MatrixIndexingCPInstruction(Operator op, CPOperand in, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr){
- super(op, in, out, opcode, istr);
- rowLower = rl;
- rowUpper = ru;
- colLower = cl;
- colUpper = cu;
+ super(op, in, rl, ru, cl, cu, out, opcode, istr);
}
public MatrixIndexingCPInstruction(Operator op, CPOperand lhsInput, CPOperand rhsInput, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr){
- super(op, lhsInput, rhsInput, out, opcode, istr);
- rowLower = rl;
- rowUpper = ru;
- colLower = cl;
- colUpper = cu;
- }
-
- public static MatrixIndexingCPInstruction parseInstruction ( String str )
- throws DMLRuntimeException {
-
- String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
- String opcode = parts[0];
-
- if ( opcode.equalsIgnoreCase("rangeReIndex") ) {
- if ( parts.length == 7 ) {
- // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
- CPOperand in, rl, ru, cl, cu, out;
- in = new CPOperand();
- rl = new CPOperand();
- ru = new CPOperand();
- cl = new CPOperand();
- cu = new CPOperand();
- out = new CPOperand();
- in.split(parts[1]);
- rl.split(parts[2]);
- ru.split(parts[3]);
- cl.split(parts[4]);
- cu.split(parts[5]);
- out.split(parts[6]);
- return new MatrixIndexingCPInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str);
- }
- else {
- throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
- }
- }
- else if ( opcode.equalsIgnoreCase("leftIndex")) {
- if ( parts.length == 8 ) {
- // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7
- CPOperand lhsInput, rhsInput, rl, ru, cl, cu, out;
- lhsInput = new CPOperand();
- rhsInput = new CPOperand();
- rl = new CPOperand();
- ru = new CPOperand();
- cl = new CPOperand();
- cu = new CPOperand();
- out = new CPOperand();
- lhsInput.split(parts[1]);
- rhsInput.split(parts[2]);
- rl.split(parts[3]);
- ru.split(parts[4]);
- cl.split(parts[5]);
- cu.split(parts[6]);
- out.split(parts[7]);
- return new MatrixIndexingCPInstruction(new SimpleOperator(null), lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, str);
- }
- else {
- throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
- }
- }
- else {
- throw new DMLRuntimeException("Unknown opcode while parsing a MatrixIndexingCPInstruction: " + str);
- }
+ super(op, lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, istr);
}
@Override
@@ -129,12 +46,7 @@ public class MatrixIndexingCPInstruction extends UnaryCPInstruction
throws DMLUnsupportedOperationException, DMLRuntimeException
{
String opcode = getOpcode();
-
- //get indexing range
- int rl = (int)(ec.getScalarInput(rowLower.getName(), rowLower.getValueType(), rowLower.isLiteral()).getLongValue()-1);
- int ru = (int)(ec.getScalarInput(rowUpper.getName(), rowUpper.getValueType(), rowUpper.isLiteral()).getLongValue()-1);
- int cl = (int)(ec.getScalarInput(colLower.getName(), colLower.getValueType(), colLower.isLiteral()).getLongValue()-1);
- int cu = (int)(ec.getScalarInput(colUpper.getName(), colUpper.getValueType(), colUpper.isLiteral()).getLongValue()-1);
+ IndexRange ixrange = getIndexRange(ec);
//get original matrix
MatrixObject mo = (MatrixObject)ec.getVariable(input1.getName());
@@ -145,12 +57,12 @@ public class MatrixIndexingCPInstruction extends UnaryCPInstruction
MatrixBlock resultBlock = null;
if( mo.isPartitioned() ) //via data partitioning
- resultBlock = mo.readMatrixPartition( new IndexRange(rl+1,ru+1,cl+1,cu+1) );
+ resultBlock = mo.readMatrixPartition(ixrange.add(1));
else //via slicing the in-memory matrix
{
//execute right indexing operation
MatrixBlock matBlock = ec.getMatrixInput(input1.getName());
- resultBlock = matBlock.sliceOperations(rl, ru, cl, cu, new MatrixBlock());
+ resultBlock = matBlock.sliceOperations(ixrange, new MatrixBlock());
//unpin rhs input
ec.releaseMatrixInput(input1.getName());
@@ -173,7 +85,6 @@ public class MatrixIndexingCPInstruction extends UnaryCPInstruction
Statistics.incrementTotalLixUIP();
Statistics.incrementTotalLix();
}
-
MatrixBlock matBlock = ec.getMatrixInput(input1.getName());
MatrixBlock resultBlock = null;
@@ -181,15 +92,16 @@ public class MatrixIndexingCPInstruction extends UnaryCPInstruction
if(input2.getDataType() == DataType.MATRIX) //MATRIX<-MATRIX
{
MatrixBlock rhsMatBlock = ec.getMatrixInput(input2.getName());
- resultBlock = matBlock.leftIndexingOperations(rhsMatBlock, rl, ru, cl, cu, new MatrixBlock(), inplace);
+ resultBlock = matBlock.leftIndexingOperations(rhsMatBlock, ixrange, new MatrixBlock(), inplace);
ec.releaseMatrixInput(input2.getName());
}
else //MATRIX<-SCALAR
{
- if(!(rl==ru && cl==cu))
- throw new DMLRuntimeException("Invalid index range of scalar leftindexing: ["+rl+":"+ru+","+cl+":"+cu+"]." );
+ if(!ixrange.isScalar())
+ throw new DMLRuntimeException("Invalid index range of scalar leftindexing: "+ixrange.toString()+"." );
ScalarObject scalar = ec.getScalarInput(input2.getName(), ValueType.DOUBLE, input2.isLiteral());
- resultBlock = (MatrixBlock) matBlock.leftIndexingOperations(scalar, rl, cl, new MatrixBlock(), inplace);
+ resultBlock = (MatrixBlock) matBlock.leftIndexingOperations(scalar,
+ (int)ixrange.rowStart, (int)ixrange.colStart, new MatrixBlock(), inplace);
}
//unpin lhs input
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarAppendCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarAppendCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarAppendCPInstruction.java
new file mode 100644
index 0000000..2bfae2d
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarAppendCPInstruction.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.DMLUnsupportedOperationException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+
+public final class ScalarAppendCPInstruction extends AppendCPInstruction
+{
+ public ScalarAppendCPInstruction(Operator op, CPOperand in1, CPOperand in2, CPOperand in3, CPOperand out, AppendType type, String opcode, String istr) {
+ super(op, in1, in2, in3, out, type, opcode, istr);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec)
+ throws DMLUnsupportedOperationException, DMLRuntimeException
+ {
+ //Append type: STRING
+
+ //get input strings (vars or literals)
+ ScalarObject so1 = ec.getScalarInput( input1.getName(), input1.getValueType(), input1.isLiteral() );
+ ScalarObject so2 = ec.getScalarInput( input2.getName(), input2.getValueType(), input2.isLiteral() );
+
+ //pre-checks
+ String val1 = so1.getStringValue();
+ String val2 = so2.getStringValue();
+ StringObject.checkMaxStringLength( val1.length()+val2.length() );
+
+ //core execution
+ String outString = val1 + "\n" + val2;
+ ScalarObject sores = new StringObject(outString);
+
+ //set output
+ ec.setScalarOutput(output.getName(), sores);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
index 352278c..f5f2b58 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
@@ -26,7 +26,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
-import org.apache.sysml.runtime.instructions.cp.MatrixIndexingCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.IndexingCPInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -43,23 +43,19 @@ import org.apache.sysml.runtime.util.MapReduceTool;
* COLWISE partition formats.
*
*/
-public class MatrixIndexingCPFileInstruction extends MatrixIndexingCPInstruction
+public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
{
-
- public MatrixIndexingCPFileInstruction(Operator op, CPOperand in, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr)
- {
+ public MatrixIndexingCPFileInstruction(Operator op, CPOperand in, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr) {
super( op, in, rl, ru, cl, cu, out, opcode, istr );
}
- public MatrixIndexingCPFileInstruction(Operator op, CPOperand lhsInput, CPOperand rhsInput, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr)
- {
+ public MatrixIndexingCPFileInstruction(Operator op, CPOperand lhsInput, CPOperand rhsInput, CPOperand rl, CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr) {
super( op, lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, istr);
}
public static MatrixIndexingCPFileInstruction parseInstruction ( String str )
throws DMLRuntimeException
- {
-
+ {
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
String opcode = parts[0];
@@ -100,17 +96,14 @@ public class MatrixIndexingCPFileInstruction extends MatrixIndexingCPInstruction
throws DMLUnsupportedOperationException, DMLRuntimeException
{
String opcode = getOpcode();
- long rl = ec.getScalarInput(rowLower.getName(), rowLower.getValueType(), rowLower.isLiteral()).getLongValue();
- long ru = ec.getScalarInput(rowUpper.getName(), rowUpper.getValueType(), rowUpper.isLiteral()).getLongValue();
- long cl = ec.getScalarInput(colLower.getName(), colLower.getValueType(), colLower.isLiteral()).getLongValue();
- long cu = ec.getScalarInput(colUpper.getName(), colUpper.getValueType(), colUpper.isLiteral()).getLongValue();
+ IndexRange ixrange = getIndexRange(ec).add(1);
MatrixObject mo = (MatrixObject) ec.getVariable(input1.getName());
if( mo.isPartitioned() && opcode.equalsIgnoreCase("rangeReIndex") )
{
MatrixFormatMetaData meta = (MatrixFormatMetaData)mo.getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- String pfname = mo.getPartitionFileName( new IndexRange(rl,ru,cl,cu), mc.getRowsPerBlock(), mc.getColsPerBlock());
+ String pfname = mo.getPartitionFileName( ixrange, mc.getRowsPerBlock(), mc.getColsPerBlock());
if( MapReduceTool.existsFileOnHDFS(pfname) )
{
@@ -142,7 +135,7 @@ public class MatrixIndexingCPFileInstruction extends MatrixIndexingCPInstruction
else
{
//will return an empty matrix partition
- MatrixBlock resultBlock = mo.readMatrixPartition( new IndexRange(rl,ru,cl,cu) );
+ MatrixBlock resultBlock = mo.readMatrixPartition( ixrange );
ec.setMatrixOutput(output.getName(), resultBlock);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 7f8f601..c2afe33 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.CacheBlock;
+import org.apache.sysml.runtime.util.IndexRange;
/**
*
@@ -450,6 +451,21 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
}
/**
+ *
+ * @param ixrange
+ * @param ret
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public FrameBlock sliceOperations(IndexRange ixrange, FrameBlock ret)
+ throws DMLRuntimeException
+ {
+ return sliceOperations(
+ (int)ixrange.rowStart, (int)ixrange.rowEnd,
+ (int)ixrange.colStart, (int)ixrange.colEnd, ret);
+ }
+
+ /**
* Right indexing operations to slice a subframe out of this frame block.
* Note that the existing column value types are preserved.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 034aeba..f3190d5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -3852,6 +3852,24 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
LibMatrixMult.matrixMultPermute(this, m2, ret1, ret2);
}
+
+ /**
+ *
+ * @param rhsMatrix
+ * @param ixrange
+ * @param ret
+ * @param inplace
+ * @return
+ * @throws DMLRuntimeException
+ * @throws DMLUnsupportedOperationException
+ */
+ public MatrixBlock leftIndexingOperations(MatrixBlock rhsMatrix, IndexRange ixrange, MatrixBlock ret, boolean inplace)
+ throws DMLRuntimeException, DMLUnsupportedOperationException
+ {
+ return leftIndexingOperations(
+ rhsMatrix, (int)ixrange.rowStart, (int)ixrange.rowEnd,
+ (int)ixrange.colStart, (int)ixrange.colEnd, ret, inplace);
+ }
/**
* Method to perform leftIndexing operation for a given lower and upper bounds in row and column dimensions.
@@ -3969,6 +3987,18 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
}
/**
+ *
+ * @param ixrange
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public MatrixBlock sliceOperations(IndexRange ixrange, MatrixBlock ret) throws DMLRuntimeException {
+ return sliceOperations(
+ (int)ixrange.rowStart, (int)ixrange.rowEnd,
+ (int)ixrange.colStart, (int)ixrange.colEnd, ret);
+ }
+
+ /**
* Method to perform rangeReIndex operation for a given lower and upper bounds in row and column dimensions.
* Extracted submatrix is returned as "result". Note: This operation is now 0-based.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/IndexRange.java b/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
index 4ba6e62..08b1d58 100644
--- a/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
+++ b/src/main/java/org/apache/sysml/runtime/util/IndexRange.java
@@ -25,26 +25,35 @@ import java.io.Serializable;
public class IndexRange implements Serializable
{
private static final long serialVersionUID = 5746526303666494601L;
- public long rowStart=0;
- public long rowEnd=0;
- public long colStart=0;
- public long colEnd=0;
- public IndexRange(long rs, long re, long cs, long ce)
- {
+ public long rowStart = 0;
+ public long rowEnd = 0;
+ public long colStart = 0;
+ public long colEnd = 0;
+
+ public IndexRange(long rs, long re, long cs, long ce) {
set(rs, re, cs, ce);
}
- public void set(long rs, long re, long cs, long ce)
- {
+
+ public void set(long rs, long re, long cs, long ce) {
rowStart = rs;
rowEnd = re;
colStart = cs;
colEnd = ce;
}
+ public boolean isScalar() {
+ return (rowStart==rowEnd && colStart==colEnd);
+ }
+
+ public IndexRange add(int delta) {
+ return new IndexRange(
+ rowStart + delta, rowEnd + delta,
+ colStart + delta, colEnd + delta);
+ }
+
@Override
- public String toString()
- {
- return "["+rowStart+":"+rowEnd+", "+colStart+":"+colEnd+"]";
+ public String toString() {
+ return "["+rowStart+":"+rowEnd+","+colStart+":"+colEnd+"]";
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameIndexingAppendTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameIndexingAppendTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameIndexingAppendTest.java
new file mode 100644
index 0000000..3b778c5
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameIndexingAppendTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.jmlc;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.jmlc.Connection;
+import org.apache.sysml.api.jmlc.PreparedScript;
+import org.apache.sysml.api.jmlc.ResultVariables;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ *
+ *
+ */
+public class FrameIndexingAppendTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME1 = "transform4";
+ private final static String TEST_DIR = "functions/jmlc/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + FrameIndexingAppendTest.class.getSimpleName() + "/";
+
+ private final static int rows = 700;
+ private final static int cols = 3;
+
+ private final static int nRuns = 2;
+
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.1;
+
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "F2" }) );
+ }
+
+ @Test
+ public void testJMLCTransformDense() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, false, false);
+ }
+
+ @Test
+ public void testJMLCTransformSparse() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, true, false);
+ }
+
+ @Test
+ public void testJMLCTransformDenseReuse() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, false, true);
+ }
+
+ @Test
+ public void testJMLCTransformSparseReuse() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, true, true);
+ }
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ * @throws IOException
+ */
+ private void runJMLCReuseTest( String testname, boolean sparse, boolean modelReuse )
+ throws IOException
+ {
+ String TEST_NAME = testname;
+
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ //generate inputs
+ double[][] Fd = TestUtils.round(getRandomMatrix(rows, cols, 0.51, 7.49, sparse?sparsity2:sparsity1, 1234));
+ String[][] F1s = FrameTransformTest.createFrameData(Fd);
+ String[][] Ms = FrameTransformTest.createRecodeMaps(F1s);
+
+ //run DML via JMLC
+ ArrayList<String[][]> F2set = execDMLScriptviaJMLC( TEST_NAME, F1s, Ms, modelReuse );
+
+ //check correct result
+ for( String[][] data : F2set )
+ for( int i=0; i<F1s.length; i++ )
+ for( int j=0; j<F1s[i].length; j++ )
+ Assert.assertEquals("Wrong result: "+data[i][j]+".", data[i][j], F1s[i][j]);
+ }
+
+ /**
+ *
+ * @param X
+ * @return
+ * @throws DMLException
+ * @throws IOException
+ */
+ private ArrayList<String[][]> execDMLScriptviaJMLC( String testname, String[][] F1, String[][] M, boolean modelReuse)
+ throws IOException
+ {
+ Timing time = new Timing(true);
+
+ ArrayList<String[][]> ret = new ArrayList<String[][]>();
+
+ //establish connection to SystemML
+ Connection conn = new Connection();
+
+ try
+ {
+ //prepare input arguments
+ HashMap<String,String> args = new HashMap<String,String>();
+ args.put("$TRANSFORM_SPEC1", "{ \"ids\": true ,\"recode\": [ 1, 2] }");
+ args.put("$TRANSFORM_SPEC2", "{ \"ids\": true ,\"recode\": [ 1] }");
+
+ //read and precompile script
+ String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
+ PreparedScript pstmt = conn.prepareScript(script, args, new String[]{"F1","M"}, new String[]{"F2"}, false);
+
+ if( modelReuse )
+ pstmt.setFrame("M", M, true);
+
+ //execute script multiple times
+ for( int i=0; i<nRuns; i++ )
+ {
+ //bind input parameters
+ if( !modelReuse )
+ pstmt.setFrame("M", M);
+ pstmt.setFrame("F1", F1);
+
+ //execute script
+ ResultVariables rs = pstmt.executeScript();
+
+ //get output parameter
+ String[][] Y = rs.getFrame("F2");
+ ret.add(Y); //keep result for comparison
+ }
+ }
+ catch(Exception ex)
+ {
+ ex.printStackTrace();
+ throw new IOException(ex);
+ }
+ finally
+ {
+ if( conn != null )
+ conn.close();
+ }
+
+ System.out.println("JMLC scoring w/ "+nRuns+" runs in "+time.stop()+"ms.");
+
+ return ret;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/36612f98/src/test/scripts/functions/jmlc/transform4.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/jmlc/transform4.dml b/src/test/scripts/functions/jmlc/transform4.dml
new file mode 100644
index 0000000..c57f8e3
--- /dev/null
+++ b/src/test/scripts/functions/jmlc/transform4.dml
@@ -0,0 +1,46 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+#read data frame and meta data
+F1 = read($F1, data_type="frame", format="csv"); #new data
+M = read($M, data_type="frame", format="csv"); #existing recode maps
+specJson1 = $TRANSFORM_SPEC1
+specJson2 = $TRANSFORM_SPEC2
+
+#frame indexing
+F11 = F1[,1:2];
+F12 = F1[,3];
+M1 = M[,1:2];
+M2 = M[,3];
+
+X1 = transformapply(target=F11, meta=M1, spec=specJson1);
+X2 = transformapply(target=F12, meta=M2, spec=specJson2);
+
+X1 = X1 * (X1!=77.7);
+X2 = X2 * (X2!=77.7);
+
+F21 = transformdecode(target=X1, meta=M1, spec=specJson1);
+F22 = transformdecode(target=X2, meta=M2, spec=specJson2);
+
+#frame append
+F2 = append(F21, F22);
+
+write(F2, $F2);