You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/05/10 19:30:28 UTC
[1/4] systemml git commit: [SYSTEMML-2308, 2309] New list data type,
incl length, rix, cast ops
Repository: systemml
Updated Branches:
refs/heads/master 42359f11c -> 6f2c885e8
[SYSTEMML-2308,2309] New list data type, incl length, rix, cast ops
This patch introduces a new list data type to group arbitrary data
elements (matrices, frames, scalars, and lists) into a single handle.
Since it's just a meta data handle, we only support the operations
length (number of entries), right indexing (point and range lookups),
casts to matrices, frames, and scalars, and rmvar instructions.
Note, that this also includes a grammar change and thus regeneration of
the antlr parser.
Furthermore, this includes a minor fix for exist operations to correctly
handle lists and frames.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d6e2ae66
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d6e2ae66
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d6e2ae66
Branch: refs/heads/master
Commit: d6e2ae66260fdab53f34c9c38f09d37d179d6e6a
Parents: 42359f1
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 9 20:40:14 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu May 10 12:28:30 2018 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 10 +-
.../java/org/apache/sysml/hops/IndexingOp.java | 3 +
src/main/java/org/apache/sysml/hops/NaryOp.java | 5 +-
.../java/org/apache/sysml/hops/UnaryOp.java | 13 ++-
...RewriteElementwiseMultChainOptimization.java | 11 +-
src/main/java/org/apache/sysml/lops/Nary.java | 3 +-
.../java/org/apache/sysml/lops/compile/Dag.java | 2 +-
.../sysml/parser/BuiltinFunctionExpression.java | 26 ++++-
.../org/apache/sysml/parser/DMLTranslator.java | 5 +
.../org/apache/sysml/parser/Expression.java | 8 +-
.../java/org/apache/sysml/parser/dml/Dml.g4 | 2 +-
.../context/ExecutionContext.java | 4 +-
.../sysml/runtime/functionobjects/Builtin.java | 4 +-
.../instructions/CPInstructionParser.java | 11 +-
.../cp/AggregateUnaryCPInstruction.java | 2 +-
.../cp/BuiltinNaryCPInstruction.java | 2 +-
.../instructions/cp/IndexingCPInstruction.java | 4 +-
.../cp/ListIndexingCPInstruction.java | 80 ++++++++++++++
.../runtime/instructions/cp/ListObject.java | 106 +++++++++++++++++++
.../cp/ScalarBuiltinNaryCPInstruction.java | 30 ++++--
.../instructions/cp/VariableCPInstruction.java | 26 +++--
.../functions/misc/ListAndStructTest.java | 97 +++++++++++++++++
src/test/scripts/functions/misc/ListUnnamed.R | 42 ++++++++
src/test/scripts/functions/misc/ListUnnamed.dml | 38 +++++++
.../functions/misc/ZPackageSuite.java | 1 +
25 files changed, 483 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 059132d..bd906b5 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -610,8 +610,7 @@ public abstract class Hop implements ParseInfo
switch( getDataType() )
{
- case SCALAR:
- {
+ case SCALAR: {
//memory estimate always known
if( getValueType()== ValueType.DOUBLE) //default case
_outputMemEstimate = OptimizerUtils.DOUBLE_SIZE;
@@ -666,8 +665,8 @@ public abstract class Hop implements ParseInfo
break;
}
case OBJECT:
- case UNKNOWN:
- {
+ case UNKNOWN:
+ case LIST: {
//memory estimate always unknown
_outputMemEstimate = OptimizerUtils.DEFAULT_SIZE;
break;
@@ -1088,7 +1087,7 @@ public abstract class Hop implements ParseInfo
// Operations that require a variable number of operands
public enum OpOpN {
- PRINTF, CBIND, RBIND, EVAL
+ PRINTF, CBIND, RBIND, EVAL, LIST,
}
public enum AggOp {
@@ -1375,6 +1374,7 @@ public abstract class Hop implements ParseInfo
HopsOpOpNLops.put(OpOpN.CBIND, Nary.OperationType.CBIND);
HopsOpOpNLops.put(OpOpN.RBIND, Nary.OperationType.RBIND);
HopsOpOpNLops.put(OpOpN.EVAL, Nary.OperationType.EVAL);
+ HopsOpOpNLops.put(OpOpN.LIST, Nary.OperationType.LIST);
}
protected static final HashMap<Hop.OpOp1, String> HopsOpOp12String;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/hops/IndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java b/src/main/java/org/apache/sysml/hops/IndexingOp.java
index c265d59..2f52f29 100644
--- a/src/main/java/org/apache/sysml/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java
@@ -388,6 +388,9 @@ public class IndexingOp extends Hop
checkAndSetInvalidCPDimsAndSize();
}
+ if( getInput().get(0).getDataType()==DataType.LIST )
+ _etype = ExecType.CP;
+
//mark for recompile (forever)
setRequiresRecompileIfNecessary();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/hops/NaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/NaryOp.java b/src/main/java/org/apache/sysml/hops/NaryOp.java
index 4b950de..a08f86c 100644
--- a/src/main/java/org/apache/sysml/hops/NaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/NaryOp.java
@@ -155,7 +155,7 @@ public class NaryOp extends Hop {
setRequiresRecompileIfNecessary();
//ensure cp exec type for single-node operations
- if ( _op == OpOpN.PRINTF || _op == OpOpN.EVAL)
+ if ( _op == OpOpN.PRINTF || _op == OpOpN.EVAL || _op == OpOpN.LIST)
_etype = ExecType.CP;
return _etype;
@@ -182,6 +182,9 @@ public class NaryOp extends Hop {
setDim1(HopRewriteUtils.getSumValidInputDims(this, true));
setDim2(HopRewriteUtils.getMaxInputDim(this, false));
break;
+ case LIST:
+ setDim1(getInput().size());
+ setDim2(1);
case PRINTF:
case EVAL:
//do nothing:
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 7b7c685..46a5f8e 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -685,8 +685,9 @@ public class UnaryOp extends Hop implements MultiThreadedHop
setRequiresRecompileIfNecessary();
//ensure cp exec type for single-node operations
- if( _op == OpOp1.PRINT || _op == OpOp1.ASSERT || _op == OpOp1.STOP
- || _op == OpOp1.INVERSE || _op == OpOp1.EIGEN || _op == OpOp1.CHOLESKY || _op == OpOp1.SVD)
+ if( _op == OpOp1.PRINT || _op == OpOp1.ASSERT || _op == OpOp1.STOP
+ || _op == OpOp1.INVERSE || _op == OpOp1.EIGEN || _op == OpOp1.CHOLESKY || _op == OpOp1.SVD
+ || getInput().get(0).getDataType() == DataType.LIST )
{
_etype = ExecType.CP;
}
@@ -697,10 +698,14 @@ public class UnaryOp extends Hop implements MultiThreadedHop
@Override
public void refreshSizeInformation()
{
- if ( getDataType() == DataType.SCALAR )
- {
+ if ( getDataType() == DataType.SCALAR ) {
//do nothing always known
}
+ else if( (_op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_FRAME
+ || _op == OpOp1.CAST_AS_SCALAR) && getInput().get(0).getDataType()==DataType.LIST ){
+ setDim1( -1 );
+ setDim2( -1 );
+ }
else if( (_op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_FRAME)
&& getInput().get(0).getDataType()==DataType.SCALAR )
{
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
index a5d8b06..59092d5 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
@@ -257,11 +257,12 @@ public class RewriteElementwiseMultChainOptimization extends HopRewriteRule {
{
for (int i = 0, valuesLength = Expression.DataType.values().length; i < valuesLength; i++)
switch(Expression.DataType.values()[i]) {
- case SCALAR: orderDataType[i] = 0; break;
- case MATRIX: orderDataType[i] = 1; break;
- case FRAME: orderDataType[i] = 2; break;
- case OBJECT: orderDataType[i] = 3; break;
- case UNKNOWN:orderDataType[i] = 4; break;
+ case SCALAR: orderDataType[i] = 0; break;
+ case MATRIX: orderDataType[i] = 1; break;
+ case FRAME: orderDataType[i] = 2; break;
+ case OBJECT: orderDataType[i] = 3; break;
+ case UNKNOWN:orderDataType[i] = 4; break;
+ case LIST: orderDataType[i] = 5; break;
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/lops/Nary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Nary.java b/src/main/java/org/apache/sysml/lops/Nary.java
index 7541ffa..f7d5952 100644
--- a/src/main/java/org/apache/sysml/lops/Nary.java
+++ b/src/main/java/org/apache/sysml/lops/Nary.java
@@ -32,7 +32,7 @@ import org.apache.sysml.parser.Expression.ValueType;
public class Nary extends Lop {
public enum OperationType {
- PRINTF, CBIND, RBIND, EVAL
+ PRINTF, CBIND, RBIND, EVAL, LIST
}
private OperationType operationType;
@@ -119,6 +119,7 @@ public class Nary extends Lop {
case CBIND:
case RBIND:
case EVAL:
+ case LIST:
return operationType.name().toLowerCase();
default:
throw new UnsupportedOperationException(
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/Dag.java b/src/main/java/org/apache/sysml/lops/compile/Dag.java
index f0f74cc..3244a3b 100644
--- a/src/main/java/org/apache/sysml/lops/compile/Dag.java
+++ b/src/main/java/org/apache/sysml/lops/compile/Dag.java
@@ -2276,7 +2276,7 @@ public class Dag<N extends Lop>
// since outputs are explicitly specified
if (node.getExecLocation() != ExecLocation.Data )
{
- if (node.getDataType() == DataType.SCALAR) {
+ if (node.getDataType() == DataType.SCALAR || node.getDataType() == DataType.LIST) {
oparams.setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
out.setVarName(oparams.getLabel());
Instruction currInstr = VariableCPInstruction.prepareRemoveInstruction(oparams.getLabel());
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
index 8ce197f..6e48839 100644
--- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.HashSet;
import org.antlr.v4.runtime.ParserRuleContext;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -510,7 +511,8 @@ public class BuiltinFunctionExpression extends DataIdentifier
break;
case CAST_AS_MATRIX:
checkNumParameters(1);
- checkScalarFrameParam(getFirstExpr());
+ checkDataTypeParam(getFirstExpr(),
+ DataType.SCALAR, DataType.FRAME, DataType.LIST);
output.setDataType(DataType.MATRIX);
output.setDimensions(id.getDim1(), id.getDim2());
if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR )
@@ -689,13 +691,21 @@ public class BuiltinFunctionExpression extends DataIdentifier
case NCOL:
case LENGTH:
checkNumParameters(1);
- checkMatrixFrameParam(getFirstExpr());
+ checkDataTypeParam(getFirstExpr(),
+ DataType.MATRIX, DataType.FRAME, DataType.LIST);
output.setDataType(DataType.SCALAR);
output.setDimensions(0, 0);
output.setBlockDimensions (0, 0);
output.setValueType(ValueType.INT);
break;
-
+
+ case LIST:
+ output.setDataType(DataType.LIST);
+ output.setValueType(ValueType.UNKNOWN);
+ output.setDimensions(getAllExpr().length, 1);
+ output.setBlockDimensions(-1, -1);
+ break;
+
case EXISTS:
checkNumParameters(1);
checkStringOrDataIdentifier(getFirstExpr());
@@ -704,7 +714,7 @@ public class BuiltinFunctionExpression extends DataIdentifier
output.setBlockDimensions (0, 0);
output.setValueType(ValueType.BOOLEAN);
break;
-
+
// Contingency tables
case TABLE:
@@ -1496,6 +1506,11 @@ public class BuiltinFunctionExpression extends DataIdentifier
}
}
+ protected void checkDataTypeParam(Expression e, DataType... dt) { //always unconditional
+ if( !ArrayUtils.contains(dt, e.getOutput().getDataType()) )
+ raiseValidateError("Non-matching expected data type for function "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
+ }
+
protected void checkMatrixFrameParam(Expression e) { //always unconditional
if (e.getOutput().getDataType() != DataType.MATRIX && e.getOutput().getDataType() != DataType.FRAME) {
raiseValidateError("Expecting matrix or frame parameter for function "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
@@ -1514,6 +1529,7 @@ public class BuiltinFunctionExpression extends DataIdentifier
}
}
+ @SuppressWarnings("unused")
private void checkScalarFrameParam(Expression e) { //always unconditional
if (e.getOutput().getDataType() != DataType.SCALAR && e.getOutput().getDataType() != DataType.FRAME) {
raiseValidateError("Expecting scalar parameter for function " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
@@ -1639,6 +1655,8 @@ public class BuiltinFunctionExpression extends DataIdentifier
bifop = Expression.BuiltinFunctionOp.MAX;
else if (functionName.equals("ppred"))
bifop = Expression.BuiltinFunctionOp.PPRED;
+ else if(functionName.equals("list"))
+ bifop = Expression.BuiltinFunctionOp.LIST;
else if (functionName.equals("log"))
bifop = Expression.BuiltinFunctionOp.LOG;
else if (functionName.equals("length"))
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 45d8fcb..9d83dde 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -2372,6 +2372,11 @@ public class DMLTranslator
target.getValueType(), Hop.OpOp1.LENGTH, expr) : new LiteralOp(expr.getDim1()*expr.getDim2());
break;
+ case LIST:
+ currBuiltinOp = new NaryOp(target.getName(), DataType.LIST, ValueType.UNKNOWN,
+ OpOpN.LIST, processAllExpressions(source.getAllExpr(), hops));
+ break;
+
case EXISTS:
currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(),
target.getValueType(), Hop.OpOp1.EXISTS, expr);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java
index 0381f2d..f369e19 100644
--- a/src/main/java/org/apache/sysml/parser/Expression.java
+++ b/src/main/java/org/apache/sysml/parser/Expression.java
@@ -98,7 +98,8 @@ public abstract class Expression implements ParseInfo
INTERQUANTILE,
INVERSE,
IQM,
- LENGTH,
+ LENGTH,
+ LIST,
LOG,
LU,
MAX,
@@ -180,7 +181,7 @@ public abstract class Expression implements ParseInfo
* Data types (matrix, scalar, frame, object, unknown).
*/
public enum DataType {
- MATRIX, SCALAR, FRAME, OBJECT, UNKNOWN;
+ MATRIX, SCALAR, FRAME, LIST, OBJECT, UNKNOWN;
public boolean isMatrix() {
return (this == MATRIX);
@@ -191,6 +192,9 @@ public abstract class Expression implements ParseInfo
public boolean isScalar() {
return (this == SCALAR);
}
+ public boolean isComposite() {
+ return (this == LIST);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/parser/dml/Dml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/Dml.g4 b/src/main/java/org/apache/sysml/parser/dml/Dml.g4
index 8723b3f..6846f7a 100644
--- a/src/main/java/org/apache/sysml/parser/dml/Dml.g4
+++ b/src/main/java/org/apache/sysml/parser/dml/Dml.g4
@@ -114,7 +114,7 @@ dataIdentifier returns [ org.apache.sysml.parser.common.ExpressionInfo dataInfo
} :
// ------------------------------------------
// IndexedIdentifier
- name=ID '[' (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? ']' # IndexedExpression
+ name=ID '[' (rowLower=expression (':' rowUpper=expression)?)? (',' (colLower=expression (':' colUpper=expression)?)?)? ']' # IndexedExpression
// ------------------------------------------
| ID # SimpleDataIdentifierExpression
| COMMANDLINE_NAMED_ID # CommandlineParamExpression
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index f339efb..67b2a83 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -497,7 +497,7 @@ public class ExecutionContext {
* @param varList variable list
* @return indicator vector of old cleanup state of matrix objects
*/
- public boolean[] pinVariables(ArrayList<String> varList)
+ public boolean[] pinVariables(List<String> varList)
{
//2-pass approach since multiple vars might refer to same matrix object
boolean[] varsState = new boolean[varList.size()];
@@ -535,7 +535,7 @@ public class ExecutionContext {
* @param varList variable list
* @param varsState variable state
*/
- public void unpinVariables(ArrayList<String> varList, boolean[] varsState) {
+ public void unpinVariables(List<String> varList, boolean[] varsState) {
for( int i=0; i<varList.size(); i++ ) {
Data dat = _variables.get(varList.get(i));
if( dat instanceof MatrixObject )
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
index 6a04a87..7ad0808 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
@@ -46,12 +46,11 @@ import org.apache.sysml.runtime.DMLScriptException;
*/
public class Builtin extends ValueFunction
{
-
private static final long serialVersionUID = 3836744687789840574L;
public enum BuiltinCode { SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN,
MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, ROUND, MAXINDEX, MININDEX,
- STOP, CEIL, FLOOR, CUMSUM, CUMPROD, CUMMIN, CUMMAX, INVERSE, SPROP, SIGMOID, EVAL }
+ STOP, CEIL, FLOOR, CUMSUM, CUMPROD, CUMMIN, CUMMAX, INVERSE, SPROP, SIGMOID, EVAL, LIST }
public BuiltinCode bFunc;
private static final boolean FASTMATH = true;
@@ -82,6 +81,7 @@ public class Builtin extends ValueFunction
String2BuiltinCode.put( "print" , BuiltinCode.PRINT);
String2BuiltinCode.put( "printf" , BuiltinCode.PRINTF);
String2BuiltinCode.put( "eval" , BuiltinCode.EVAL);
+ String2BuiltinCode.put( "list" , BuiltinCode.LIST);
String2BuiltinCode.put( "nrow" , BuiltinCode.NROW);
String2BuiltinCode.put( "ncol" , BuiltinCode.NCOL);
String2BuiltinCode.put( "length" , BuiltinCode.LENGTH);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index 395a4ec..9e81f08 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -179,11 +179,12 @@ public class CPInstructionParser extends InstructionParser
String2CPInstructionType.put( "sprop", CPType.Unary);
String2CPInstructionType.put( "sigmoid", CPType.Unary);
- String2CPInstructionType.put( "printf" , CPType.BuiltinNary);
- String2CPInstructionType.put( "cbind" , CPType.BuiltinNary);
- String2CPInstructionType.put( "rbind" , CPType.BuiltinNary);
- String2CPInstructionType.put( "eval" , CPType.BuiltinNary);
-
+ String2CPInstructionType.put( "printf", CPType.BuiltinNary);
+ String2CPInstructionType.put( "cbind", CPType.BuiltinNary);
+ String2CPInstructionType.put( "rbind", CPType.BuiltinNary);
+ String2CPInstructionType.put( "eval", CPType.BuiltinNary);
+ String2CPInstructionType.put( "list", CPType.BuiltinNary);
+
// Parameterized Builtin Functions
String2CPInstructionType.put( "cdf", CPType.ParameterizedBuiltin);
String2CPInstructionType.put( "invcdf", CPType.ParameterizedBuiltin);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
index 2c74a10..c9d9dc7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
@@ -123,7 +123,7 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction
}
else if( _type == AUType.EXISTS ) {
//probe existence of variable in symbol table w/o error
- String varName = input1.isMatrix() ? input1.getName() :
+ String varName = !input1.isScalar() ? input1.getName() :
ec.getScalarInput(input1).getStringValue();
boolean rval = ec.getVariables().keySet().contains(varName);
//create and set output scalar
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
index 8f5058d..eac9c32 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
@@ -58,7 +58,7 @@ public abstract class BuiltinNaryCPInstruction extends CPInstruction
inputOperands[i-1] = new CPOperand(parts[i]);
}
- if( Nary.OperationType.PRINTF.name().equalsIgnoreCase(opcode) ) {
+ if( "printf".equals(opcode) || "list".equals(opcode)) {
ValueFunction func = Builtin.getBuiltinFnObject(opcode);
return new ScalarBuiltinNaryCPInstruction(new SimpleOperator(func),
opcode, str, outputOperand, inputOperands);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
index 4413139..0046713 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
@@ -71,8 +71,10 @@ public abstract class IndexingCPInstruction extends UnaryCPInstruction {
out = new CPOperand(parts[6]);
if( in.getDataType()==DataType.MATRIX )
return new MatrixIndexingCPInstruction(in, rl, ru, cl, cu, out, opcode, str);
- else if (in.getDataType() == DataType.FRAME)
+ else if( in.getDataType() == DataType.FRAME )
return new FrameIndexingCPInstruction(in, rl, ru, cl, cu, out, opcode, str);
+ else if( in.getDataType() == DataType.LIST )
+ return new ListIndexingCPInstruction(in, rl, ru, cl, cu, out, opcode, str);
else
throw new DMLRuntimeException("Can index only on Frames or Matrices");
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java
new file mode 100644
index 0000000..59b2d48
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.util.IndexRange;
+
+public final class ListIndexingCPInstruction extends IndexingCPInstruction {
+
+ protected ListIndexingCPInstruction(CPOperand in, CPOperand rl, CPOperand ru, CPOperand cl,
+ CPOperand cu, CPOperand out, String opcode, String istr) {
+ super(in, rl, ru, cl, cu, out, opcode, istr);
+ }
+
+ protected ListIndexingCPInstruction(CPOperand lhsInput, CPOperand rhsInput, CPOperand rl,
+ CPOperand ru, CPOperand cl, CPOperand cu, CPOperand out, String opcode, String istr) {
+ super(lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, istr);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec) {
+ String opcode = getOpcode();
+ IndexRange ixrange = getIndexRange(ec);
+
+ //right indexing
+ if( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) {
+ ListObject list = (ListObject) ec.getVariable(input1.getName());
+
+ //execute right indexing operation and set output
+ ec.setVariable(output.getName(),
+ list.slice((int)ixrange.rowStart, (int)ixrange.rowEnd));
+ }
+ //left indexing
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) {
+// FrameBlock lin = ec.getFrameInput(input1.getName());
+// FrameBlock out = null;
+//
+// if(input2.getDataType() == DataType.FRAME) { //FRAME<-FRAME
+// FrameBlock rin = ec.getFrameInput(input2.getName());
+// out = lin.leftIndexingOperations(rin, ixrange, new FrameBlock());
+// ec.releaseFrameInput(input2.getName());
+// }
+// else { //FRAME<-SCALAR
+// if(!ixrange.isScalar())
+// throw new DMLRuntimeException("Invalid index range of scalar leftindexing: "+ixrange.toString()+"." );
+// ScalarObject scalar = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
+// out = new FrameBlock(lin);
+// out.set((int)ixrange.rowStart, (int)ixrange.colStart, scalar.getStringValue());
+// }
+//
+// //unpin lhs input
+// ec.releaseFrameInput(input1.getName());
+//
+// //unpin output
+// ec.setFrameOutput(output.getName(), out);
+ }
+ else
+ throw new DMLRuntimeException("Invalid opcode (" + opcode +") encountered in ListIndexingCPInstruction.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
new file mode 100644
index 0000000..f773847
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+
+public class ListObject extends Data
+{
+ private static final long serialVersionUID = 3652422061598967358L;
+
+ private final List<String> _names;
+ private final List<Data> _data;
+ private boolean[] _dataState = null;
+
+ public ListObject(List<Data> data) {
+ super(DataType.LIST, ValueType.UNKNOWN);
+ _data = data;
+ _names = null;
+ }
+
+ public ListObject(List<Data> data, List<String> names) {
+ super(DataType.LIST, ValueType.UNKNOWN);
+ _data = data;
+ _names = names;
+ }
+
+ public void setStatus(boolean[] status) {
+ _dataState = status;
+ }
+
+ public boolean[] getStatus() {
+ return _dataState;
+ }
+
+ public int getLength() {
+ return _data.size();
+ }
+
+ public Data slice(int ix) {
+ return _data.get(ix);
+ }
+
+ public ListObject slice(int ix1, int ix2) {
+ ListObject ret = new ListObject(_data.subList(ix1, ix2+1),
+ (_names!=null) ? _names.subList(ix1, ix2+1) : null);
+ ret.setStatus(Arrays.copyOfRange(_dataState, ix2, ix2+1));
+ return ret;
+ }
+
+ public Data slice(String name) {
+ //check for existing named list
+ if( _names == null )
+ throw new DMLRuntimeException("Invalid lookup by name in unnamed list.");
+
+ //find position and check for existing entry
+ int pos = _names.indexOf(name);
+ if( pos <= 0 || pos >= _data.size() )
+ throw new DMLRuntimeException("List lookup return no entry for name='"+name+"'");
+
+ //return existing entry
+ return slice(pos);
+ }
+
+ @Override
+ public String getDebugName() {
+ return toString();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("List (");
+ for( int i=0; i<_data.size(); i++ ) {
+ if( i > 0 )
+ sb.append(", ");
+ if( _names != null ) {
+ sb.append(_names.get(i));
+ sb.append("=");
+ }
+ sb.append(_data.get(i).toString());
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
index 85eec77..d2ca6f5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
@@ -20,10 +20,11 @@
package org.apache.sysml.runtime.instructions.cp;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.lops.Nary;
import org.apache.sysml.parser.Expression;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
@@ -44,7 +45,7 @@ public class ScalarBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction {
@Override
public void processInstruction(ExecutionContext ec) {
- if (Nary.OperationType.PRINTF.toString().equalsIgnoreCase(getOpcode())) {
+ if( "printf".equals(getOpcode()) ) {
List<ScalarObject> scalarObjects = new ArrayList<>();
for (CPOperand input : inputs) {
ScalarObject so = ec.getScalarInput(input.getName(), input.getValueType(), input.isLiteral());
@@ -86,13 +87,26 @@ public class ScalarBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction {
if (!DMLScript.suppressPrint2Stdout()) {
System.out.println(result);
}
-
- // this is necessary so that the remove variable operation can be
- // performed
+
ec.setScalarOutput(output.getName(), new StringObject(result));
- } else {
- throw new DMLRuntimeException(
- "Opcode (" + getOpcode() + ") not recognized in ScalarBuiltinMultipleCPInstruction");
+ }
+ else if( "list".equals(getOpcode()) ) {
+ //obtain all input data objects, incl handling of literals
+ List<Data> data = Arrays.stream(inputs)
+ .map(in -> ec.getVariable(in)).collect(Collectors.toList());
+
+ //create list object over all inputs
+ ListObject list = new ListObject(data);
+
+ //disable cleanup of individual objects and store cleanup state
+ list.setStatus(ec.pinVariables(Arrays.stream(inputs)
+ .map(in -> in.getName()).collect(Collectors.toList())));
+
+ ec.setVariable(output.getName(), list);
+ }
+ else {
+ throw new DMLRuntimeException("Opcode (" + getOpcode()
+ + ") not recognized in ScalarBuiltinMultipleCPInstruction");
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index e9a6fd0..6d30346 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -572,18 +572,28 @@ public class VariableCPInstruction extends CPInstruction {
}
break;
case CastAsMatrixVariable:{
- MatrixBlock out = null;
if( getInput1().getDataType()==DataType.FRAME ) {
FrameBlock fin = ec.getFrameInput(getInput1().getName());
- out = DataConverter.convertToMatrixBlock(fin);
+ MatrixBlock out = DataConverter.convertToMatrixBlock(fin);
ec.releaseFrameInput(getInput1().getName());
+ ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
}
- else { //assume DataType.SCALAR otherwise
- ScalarObject scalarInput = ec.getScalarInput(getInput1().getName(), getInput1().getValueType(), getInput1().isLiteral());
- out = new MatrixBlock(1,1,false);
- out.quickSetValue(0, 0, scalarInput.getDoubleValue());
+ else if( getInput1().getDataType()==DataType.SCALAR ) {
+ ScalarObject scalarInput = ec.getScalarInput(
+ getInput1().getName(), getInput1().getValueType(), getInput1().isLiteral());
+ MatrixBlock out = new MatrixBlock(1,1,false);
+ out.quickSetValue(0, 0, scalarInput.getDoubleValue());
+ ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
+ }
+ else if( getInput1().getDataType()==DataType.LIST ) {
+ //TODO handling of cleanup status, potentially new object
+ ListObject list = (ListObject)ec.getVariable(getInput1().getName());
+ ec.setVariable(output.getName(), list.slice(0));
+ }
+ else {
+ throw new DMLRuntimeException("Unsupported data type "
+ + "in as.matrix(): "+getInput1().getDataType().name());
}
- ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
break;
}
case CastAsFrameVariable:{
@@ -592,7 +602,7 @@ public class VariableCPInstruction extends CPInstruction {
ScalarObject scalarInput = ec.getScalarInput(getInput1());
out = new FrameBlock(1, getInput1().getValueType());
out.ensureAllocatedColumns(1);
- out.set(0, 0, scalarInput.getStringValue());
+ out.set(0, 0, scalarInput.getStringValue());
}
else { //DataType.FRAME
MatrixBlock min = ec.getMatrixInput(getInput1().getName(), getExtendedOpcode());
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
new file mode 100644
index 0000000..b831032
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.misc;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class ListAndStructTest extends AutomatedTestBase
+{
+ private static final String TEST_NAME1 = "ListUnnamed";
+ private static final String TEST_NAME2 = "ListNamed";
+
+ private static final String TEST_DIR = "functions/misc/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + ListAndStructTest.class.getSimpleName() + "/";
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) );
+ }
+
+ @Test
+ public void testListUnnamed() {
+ runListStructTest(TEST_NAME1, false);
+ }
+
+ @Test
+ public void testListUnnamedRewrites() {
+ runListStructTest(TEST_NAME1, true);
+ }
+
+// @Test
+// public void testListNamed() {
+// runListStructTest(TEST_NAME2, false);
+// }
+//
+// @Test
+// public void testListNamedRewrites() {
+// runListStructTest(TEST_NAME2, true);
+// }
+
+ private void runListStructTest(String testname, boolean rewrites)
+ {
+ boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+
+ try {
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{ "-stats","-explain","-args", output("R") };
+ fullRScriptName = HOME + testname + ".R";
+ rCmd = getRCmd(expectedDir());
+
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+
+ //run test
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ Assert.assertEquals(dmlfile.get(new CellIndex(1,1)), rfile.get(new CellIndex(1,1)));
+ }
+ finally {
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/test/scripts/functions/misc/ListUnnamed.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListUnnamed.R b/src/test/scripts/functions/misc/ListUnnamed.R
new file mode 100644
index 0000000..ede3c2b
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListUnnamed.R
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(A, B, c, D, matrix(5, 3, 3), 6);
+
+for( i in 1:length(X) ) {
+ tmp = X[i]
+ if( !exists("tmp") )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X[[5]])));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[1], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/test/scripts/functions/misc/ListUnnamed.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListUnnamed.dml b/src/test/scripts/functions/misc/ListUnnamed.dml
new file mode 100644
index 0000000..6cd4de5
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListUnnamed.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(A, B, c, D, matrix(5, 3, 3), 6);
+
+for( i in 1:length(X) ) {
+ tmp = X[i];
+ if( !exists(tmp) )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X[5])));
+
+write(R, $1);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6e2ae66/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
index 6166e3d..ab232d2 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java
@@ -45,6 +45,7 @@ import org.junit.runners.Suite;
IPAScalarRecursionTest.class,
IPAScalarVariablePropagationTest.class,
IPAUnknownRecursionTest.class,
+ ListAndStructTest.class,
LongOverflowTest.class,
NegativeLoopIncrementsTest.class,
NrowNcolStringTest.class,
[3/4] systemml git commit: [SYSTEMML-2308,
2310] New dynamic structs via named lists, incl rix
Posted by mb...@apache.org.
[SYSTEMML-2308,2310] New dynamic structs via named lists, incl rix
Following the introduction of list data types as a handle for arbitrary
data objects, this patch now introduced dynamic structs implemented via
named lists. Specifically, this allows the named compositions such as X
= list(a=M1, b=M2, c=7, d=M4) and indexing operations such as X[2] or
X['d'], i.e., with associative indexing by name. We also support range
based indexing such as X[1:4] or X['a':'d'] where the order and thus
value range is defined by the specification order on construction.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f9020a16
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f9020a16
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f9020a16
Branch: refs/heads/master
Commit: f9020a169b6291d18b3baa29eab7f5051bd2edf5
Parents: fa6394c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 9 23:07:47 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu May 10 12:28:48 2018 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 28 +++++++------
.../sysml/hops/ParameterizedBuiltinOp.java | 21 ++++++----
.../apache/sysml/hops/recompile/Recompiler.java | 2 +-
.../sysml/hops/rewrite/HopRewriteUtils.java | 3 +-
.../RewriteAlgebraicSimplificationDynamic.java | 7 ++--
.../RewriteAlgebraicSimplificationStatic.java | 7 ++--
.../apache/sysml/lops/ParameterizedBuiltin.java | 18 ++++++---
.../sysml/parser/BuiltinFunctionExpression.java | 5 ++-
.../org/apache/sysml/parser/DMLTranslator.java | 10 ++++-
.../org/apache/sysml/parser/Expression.java | 3 +-
.../org/apache/sysml/parser/Identifier.java | 34 +++++++---------
.../ParameterizedBuiltinFunctionExpression.java | 35 +++++++++++-----
.../instructions/CPInstructionParser.java | 1 +
.../cp/ListIndexingCPInstruction.java | 15 +++++--
.../runtime/instructions/cp/ListObject.java | 23 ++++++++++-
.../cp/ParameterizedBuiltinCPInstruction.java | 36 ++++++++++++-----
.../cp/ScalarBuiltinNaryCPInstruction.java | 2 +-
.../functions/misc/ListAndStructTest.java | 18 ++++-----
src/test/scripts/functions/misc/ListNamed.R | 42 ++++++++++++++++++++
src/test/scripts/functions/misc/ListNamed.dml | 38 ++++++++++++++++++
20 files changed, 254 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index bd906b5..a42b7ab 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -44,6 +44,7 @@ import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.Ternary;
import org.apache.sysml.lops.Unary;
import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.lops.ParameterizedBuiltin;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.parser.ParseInfo;
@@ -1116,7 +1117,7 @@ public abstract class Hop implements ParseInfo
INVALID, CDF, INVCDF, GROUPEDAGG, RMEMPTY, REPLACE, REXPAND,
LOWER_TRI, UPPER_TRI,
TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMCOLMAP, TRANSFORMMETA,
- TOSTRING
+ TOSTRING, LIST,
}
public enum FileFormatTypes {
@@ -1414,18 +1415,19 @@ public abstract class Hop implements ParseInfo
protected static final HashMap<Hop.ParamBuiltinOp, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes> HopsParameterizedBuiltinLops;
static {
HopsParameterizedBuiltinLops = new HashMap<>();
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.CDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.CDF);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.INVCDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.INVCDF);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.RMEMPTY);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REPLACE);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REXPAND);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.LOWER_TRI, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.LOWER_TRI);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.UPPER_TRI, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.UPPER_TRI);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMAPPLY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMAPPLY);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMDECODE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMDECODE);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMCOLMAP, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMCOLMAP);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMMETA, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMMETA);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TOSTRING, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TOSTRING);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.CDF, ParameterizedBuiltin.OperationTypes.CDF);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.INVCDF, ParameterizedBuiltin.OperationTypes.INVCDF);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, ParameterizedBuiltin.OperationTypes.RMEMPTY);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, ParameterizedBuiltin.OperationTypes.REPLACE);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, ParameterizedBuiltin.OperationTypes.REXPAND);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.LOWER_TRI, ParameterizedBuiltin.OperationTypes.LOWER_TRI);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.UPPER_TRI, ParameterizedBuiltin.OperationTypes.UPPER_TRI);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMAPPLY, ParameterizedBuiltin.OperationTypes.TRANSFORMAPPLY);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMDECODE, ParameterizedBuiltin.OperationTypes.TRANSFORMDECODE);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMCOLMAP, ParameterizedBuiltin.OperationTypes.TRANSFORMCOLMAP);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMMETA, ParameterizedBuiltin.OperationTypes.TRANSFORMMETA);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TOSTRING, ParameterizedBuiltin.OperationTypes.TOSTRING);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.LIST, ParameterizedBuiltin.OperationTypes.LIST);
}
protected static final HashMap<Hop.OpOp2, String> HopsOpOp2String;
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index b94ff5c..e287b20 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -20,6 +20,7 @@
package org.apache.sysml.hops;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map.Entry;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
@@ -87,7 +88,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
* @param inputParameters map of input parameters
*/
public ParameterizedBuiltinOp(String l, DataType dt, ValueType vt,
- ParamBuiltinOp op, HashMap<String, Hop> inputParameters) {
+ ParamBuiltinOp op, LinkedHashMap<String, Hop> inputParameters) {
super(l, dt, vt);
_op = op;
@@ -191,10 +192,11 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
case TRANSFORMDECODE:
case TRANSFORMCOLMAP:
case TRANSFORMMETA:
- case TOSTRING: {
+ case TOSTRING:
+ case LIST: {
ExecType et = optFindExecType();
ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,
- HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
+ HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
setOutputDimensions(pbilop);
setLineNumbers(pbilop);
setLops(pbilop);
@@ -1064,11 +1066,9 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
//force CP for in-memory only transform builtins
if( (_op == ParamBuiltinOp.TRANSFORMAPPLY && REMOTE==ExecType.MR)
|| _op == ParamBuiltinOp.TRANSFORMDECODE && REMOTE==ExecType.MR
- || _op == ParamBuiltinOp.TRANSFORMCOLMAP
- || _op == ParamBuiltinOp.TRANSFORMMETA
- || _op == ParamBuiltinOp.TOSTRING
- || _op == ParamBuiltinOp.CDF
- || _op == ParamBuiltinOp.INVCDF) {
+ || _op == ParamBuiltinOp.TRANSFORMCOLMAP || _op == ParamBuiltinOp.TRANSFORMMETA
+ || _op == ParamBuiltinOp.TOSTRING || _op == ParamBuiltinOp.LIST
+ || _op == ParamBuiltinOp.CDF || _op == ParamBuiltinOp.INVCDF) {
_etype = ExecType.CP;
}
@@ -1176,6 +1176,11 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
setDim2( 3 ); //fixed schema
break;
}
+ case LIST: {
+ setDim1( getInput().size() );
+ setDim2(1);
+ break;
+ }
default:
//do nothing
break;
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 2cfeada..fe69b49 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -1373,7 +1373,7 @@ public class Recompiler
updatedSizeExpr = initUnknown & d.dimsKnown();
}
//update size expression for indexing according to symbol table entries
- else if( hop instanceof IndexingOp )
+ else if( hop instanceof IndexingOp && hop.getDataType()!=DataType.LIST )
{
IndexingOp iop = (IndexingOp)hop;
Hop input2 = iop.getInput().get(1); //inpRowL
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index 3a0c3f2..6da1b7a 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import org.apache.commons.lang.ArrayUtils;
import org.apache.sysml.api.DMLScript;
@@ -618,7 +619,7 @@ public class HopRewriteUtils
return mmult;
}
- public static ParameterizedBuiltinOp createParameterizedBuiltinOp(Hop input, HashMap<String,Hop> args, ParamBuiltinOp op) {
+ public static ParameterizedBuiltinOp createParameterizedBuiltinOp(Hop input, LinkedHashMap<String,Hop> args, ParamBuiltinOp op) {
ParameterizedBuiltinOp pbop = new ParameterizedBuiltinOp("tmp", DataType.MATRIX, ValueType.DOUBLE, op, args);
pbop.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock());
copyLineNumbers(input, pbop);
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index 545ffd8..81c20e0 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -2542,7 +2543,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
&& HopRewriteUtils.isSizeExpressionOf(hi.getInput().get(3), second, true) )
{
//setup input parameter hops
- HashMap<String,Hop> args = new HashMap<>();
+ LinkedHashMap<String,Hop> args = new LinkedHashMap<>();
args.put("target", second);
args.put("max", hi.getInput().get(4));
args.put("dir", new LiteralOp("cols"));
@@ -2563,7 +2564,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
&& HopRewriteUtils.isSizeExpressionOf(hi.getInput().get(4), first, true) )
{
//setup input parameter hops
- HashMap<String,Hop> args = new HashMap<>();
+ LinkedHashMap<String,Hop> args = new LinkedHashMap<>();
args.put("target", first);
args.put("max", hi.getInput().get(3));
args.put("dir", new LiteralOp("rows"));
@@ -2572,7 +2573,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
//create new hop
ParameterizedBuiltinOp pbop = HopRewriteUtils
- .createParameterizedBuiltinOp(first, args, ParamBuiltinOp.REXPAND);
+ .createParameterizedBuiltinOp(first, args, ParamBuiltinOp.REXPAND);
HopRewriteUtils.replaceChildReference(parent, hi, pbop, pos);
HopRewriteUtils.cleanupUnreferenced(hi);
hi = pbop;
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index 4eb223d..db28a18 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -1568,7 +1569,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
&& HopRewriteUtils.isBinaryMatrixScalar(hi.getInput().get(1).getInput().get(0), OpOp2.EQUAL, 0)
&& hi.getInput().get(1).getInput().get(0).getInput().contains(hi.getInput().get(0)) )
{
- HashMap<String, Hop> args = new HashMap<>();
+ LinkedHashMap<String, Hop> args = new LinkedHashMap<>();
args.put("target", hi.getInput().get(0));
args.put("pattern", new LiteralOp(0));
args.put("replacement", hi.getInput().get(1).getInput().get(1));
@@ -1785,7 +1786,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
String direction = HopRewriteUtils.isBasic1NSequence(hi.getInput().get(0)) ? "rows" : "cols";
//setup input parameter hops
- HashMap<String,Hop> inputargs = new HashMap<>();
+ LinkedHashMap<String,Hop> inputargs = new LinkedHashMap<>();
inputargs.put("target", trgt);
inputargs.put("max", HopRewriteUtils.getBasic1NSequenceMax(seq));
inputargs.put("dir", new LiteralOp(direction));
@@ -1800,7 +1801,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
HopRewriteUtils.replaceChildReference(parent, hi, pbop, pos);
hi = pbop;
- LOG.debug("Applied simplifyOuterSeqExpand (line "+hi.getBeginLine()+")");
+ LOG.debug("Applied simplifyOuterSeqExpand (line "+hi.getBeginLine()+")");
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java b/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
index f011ba4..898d875 100644
--- a/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
+++ b/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
@@ -38,7 +38,7 @@ public class ParameterizedBuiltin extends Lop
public enum OperationTypes {
CDF, INVCDF, RMEMPTY, REPLACE, REXPAND, LOWER_TRI, UPPER_TRI,
TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMCOLMAP, TRANSFORMMETA,
- TOSTRING
+ TOSTRING, LIST
}
private OperationTypes _operation;
@@ -215,13 +215,19 @@ public class ParameterizedBuiltin extends Lop
case TRANSFORMAPPLY:
case TRANSFORMDECODE:
case TRANSFORMCOLMAP:
- case TRANSFORMMETA: {
- sb.append(_operation.toString().toLowerCase()); //opcode
+ case TRANSFORMMETA:{
+ sb.append(_operation.name().toLowerCase()); //opcode
sb.append(OPERAND_DELIMITOR);
sb.append(compileGenericParamMap(_inputParams));
break;
- }
- case TOSTRING:{
+ }
+ case LIST: {
+ sb.append("nvlist"); //opcode
+ sb.append(OPERAND_DELIMITOR);
+ sb.append(compileGenericParamMap(_inputParams));
+ break;
+ }
+ case TOSTRING: {
sb.append("toString"); //opcode
sb.append(OPERAND_DELIMITOR);
sb.append(compileGenericParamMap(_inputParams));
@@ -232,7 +238,7 @@ public class ParameterizedBuiltin extends Lop
throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
}
- if (_operation == OperationTypes.RMEMPTY) {
+ if (_operation == OperationTypes.RMEMPTY) {
sb.append("bRmEmptyBC");
sb.append(NAME_VALUE_SEPARATOR);
sb.append( _bRmEmptyBC );
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
index 6e48839..0e949d0 100644
--- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
@@ -1655,7 +1655,8 @@ public class BuiltinFunctionExpression extends DataIdentifier
bifop = Expression.BuiltinFunctionOp.MAX;
else if (functionName.equals("ppred"))
bifop = Expression.BuiltinFunctionOp.PPRED;
- else if(functionName.equals("list"))
+ else if(functionName.equals("list") //unnamed list
+ && paramExprsPassed.stream().allMatch(p -> p.getName()==null))
bifop = Expression.BuiltinFunctionOp.LIST;
else if (functionName.equals("log"))
bifop = Expression.BuiltinFunctionOp.LOG;
@@ -1682,7 +1683,7 @@ public class BuiltinFunctionExpression extends DataIdentifier
else if (functionName.equals("t"))
bifop = Expression.BuiltinFunctionOp.TRANS;
else if (functionName.equals("rev"))
- bifop = Expression.BuiltinFunctionOp.REV;
+ bifop = Expression.BuiltinFunctionOp.REV;
else if (functionName.equals("cbind") || functionName.equals("append"))
bifop = Expression.BuiltinFunctionOp.CBIND;
else if (functionName.equals("rbind"))
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 9d83dde..73a3a34 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -1962,7 +1963,7 @@ public class DMLTranslator
}
}
- private static Hop constructDfHop(String name, DataType dt, ValueType vt, ParameterizedBuiltinFunctionOp op, HashMap<String,Hop> paramHops) {
+ private static Hop constructDfHop(String name, DataType dt, ValueType vt, ParameterizedBuiltinFunctionOp op, LinkedHashMap<String,Hop> paramHops) {
// Add a hop to paramHops to store distribution information.
// Distribution parameter hops would have been already present in paramHops.
@@ -2055,7 +2056,7 @@ public class DMLTranslator
HashMap<String, Hop> hops) {
// this expression has multiple "named" parameters
- HashMap<String, Hop> paramHops = new HashMap<>();
+ LinkedHashMap<String, Hop> paramHops = new LinkedHashMap<>();
// -- construct hops for all input parameters
// -- store them in hashmap so that their "name"s are maintained
@@ -2120,6 +2121,11 @@ public class DMLTranslator
HopRewriteUtils.createBinary(paramHops.get("target"), new LiteralOp(""), OpOp2.PLUS);
break;
+ case LIST:
+ currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(),
+ target.getValueType(), ParamBuiltinOp.LIST, paramHops);
+ break;
+
default:
throw new ParseException(source.printErrorLocation() +
"processParameterizedBuiltinFunctionExpression() -- Unknown operation: " + source.getOpCode());
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java
index f369e19..66f08c5 100644
--- a/src/main/java/org/apache/sysml/parser/Expression.java
+++ b/src/main/java/org/apache/sysml/parser/Expression.java
@@ -159,7 +159,8 @@ public abstract class Expression implements ParseInfo
// Distribution Functions
CDF, INVCDF, PNORM, QNORM, PT, QT, PF, QF, PCHISQ, QCHISQ, PEXP, QEXP,
TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMENCODE, TRANSFORMCOLMAP, TRANSFORMMETA,
- TOSTRING, // The "toString" method for DML; named arguments accepted to format output
+ TOSTRING, // The "toString" method for DML; named arguments accepted to format output
+ LIST, // named argument lists; unnamed lists become builtin function
INVALID
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/parser/Identifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Identifier.java b/src/main/java/org/apache/sysml/parser/Identifier.java
index ffb455b..bf36a2f 100644
--- a/src/main/java/org/apache/sysml/parser/Identifier.java
+++ b/src/main/java/org/apache/sysml/parser/Identifier.java
@@ -161,47 +161,43 @@ public abstract class Identifier extends Expression
if (indexedIdentiferOut.getRowLowerBound() != null) {
indexedIdentiferOut.getRowLowerBound().validateExpression(ids, constVars, conditional);
-
Expression tempExpr = indexedIdentiferOut.getRowLowerBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
+ if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
raiseValidateError("Matrix values for row lower index bound are not supported, which includes indexed identifiers.", conditional);
}
-
}
+
if (indexedIdentiferOut.getRowUpperBound() != null) {
indexedIdentiferOut.getRowUpperBound().validateExpression(ids, constVars, conditional);
-
Expression tempExpr = indexedIdentiferOut.getRowUpperBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
+ if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
raiseValidateError("Matrix values for row upper index bound are not supported, which includes indexed identifiers.", conditional);
}
-
}
- if (indexedIdentiferOut.getColLowerBound() != null) {
- indexedIdentiferOut.getColLowerBound().validateExpression(ids,constVars, conditional);
+ if (indexedIdentiferOut.getColLowerBound() != null) {
+ indexedIdentiferOut.getColLowerBound().validateExpression(ids,constVars, conditional);
Expression tempExpr = indexedIdentiferOut.getColLowerBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
+ if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
raiseValidateError("Matrix values for column lower index bound are not supported, which includes indexed identifiers.", conditional);
}
-
}
+
if (indexedIdentiferOut.getColUpperBound() != null) {
indexedIdentiferOut.getColUpperBound().validateExpression(ids, constVars, conditional);
-
- Expression tempExpr = indexedIdentiferOut.getColUpperBound();
- if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
+ Expression tempExpr = indexedIdentiferOut.getColUpperBound();
+ if (tempExpr.getOutput().getDataType() == Expression.DataType.MATRIX){
raiseValidateError("Matrix values for column upper index bound are not supported, which includes indexed identifiers.", conditional);
}
-
}
- IndexPair updatedIndices = ((IndexedIdentifier)this.getOutput()).calculateIndexedDimensions(ids, constVars, conditional);
- ((IndexedIdentifier)this.getOutput()).setDimensions(updatedIndices._row, updatedIndices._col);
-
+ if( this.getOutput().getDataType() != DataType.LIST ) {
+ IndexPair updatedIndices = ((IndexedIdentifier)this.getOutput()).calculateIndexedDimensions(ids, constVars, conditional);
+ ((IndexedIdentifier)this.getOutput()).setDimensions(updatedIndices._row, updatedIndices._col);
+ }
}
-
- } else {
+ }
+ else {
this.getOutput().setProperties(this.getOutput());
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
index 6f9d6f7..ffc8bc6 100644
--- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.Set;
import java.util.stream.Collectors;
@@ -35,9 +36,10 @@ import org.apache.wink.json4j.JSONObject;
public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
{
-
+ //note: we use a linked hashmap to preserve the order of
+ //parameters if needed, such as for named lists
private ParameterizedBuiltinFunctionOp _opcode;
- private HashMap<String,Expression> _varParams;
+ private LinkedHashMap<String,Expression> _varParams;
public static final String TF_FN_PARAM_DATA = "target";
public static final String TF_FN_PARAM_MTD2 = "meta";
@@ -79,6 +81,7 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
// toString
opcodeMap.put("toString", Expression.ParameterizedBuiltinFunctionOp.TOSTRING);
+ opcodeMap.put("list", Expression.ParameterizedBuiltinFunctionOp.LIST);
}
public static HashMap<Expression.ParameterizedBuiltinFunctionOp, ParamBuiltinOp> pbHopMap;
@@ -123,17 +126,17 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
if ( pbifop == null )
return null;
- HashMap<String,Expression> varParams = new HashMap<>();
+ LinkedHashMap<String,Expression> varParams = new LinkedHashMap<>();
for (ParameterExpression pexpr : paramExprsPassed)
varParams.put(pexpr.getName(), pexpr.getExpr());
- ParameterizedBuiltinFunctionExpression retVal = new ParameterizedBuiltinFunctionExpression(ctx, pbifop,
- varParams, fileName);
+ ParameterizedBuiltinFunctionExpression retVal =
+ new ParameterizedBuiltinFunctionExpression(ctx, pbifop,varParams, fileName);
return retVal;
}
- public ParameterizedBuiltinFunctionExpression(ParserRuleContext ctx, ParameterizedBuiltinFunctionOp op, HashMap<String,Expression> varParams,
+ public ParameterizedBuiltinFunctionExpression(ParserRuleContext ctx, ParameterizedBuiltinFunctionOp op, LinkedHashMap<String,Expression> varParams,
String filename) {
_opcode = op;
_varParams = varParams;
@@ -141,7 +144,7 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
}
public ParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionOp op,
- HashMap<String, Expression> varParams, ParseInfo parseInfo) {
+ LinkedHashMap<String, Expression> varParams, ParseInfo parseInfo) {
_opcode = op;
_varParams = varParams;
setParseInfo(parseInfo);
@@ -149,7 +152,7 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
@Override
public Expression rewriteExpression(String prefix) {
- HashMap<String,Expression> newVarParams = new HashMap<>();
+ LinkedHashMap<String,Expression> newVarParams = new LinkedHashMap<>();
for (String key : _varParams.keySet()){
Expression newExpr = _varParams.get(key).rewriteExpression(prefix);
newVarParams.put(key, newExpr);
@@ -258,7 +261,11 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
case TOSTRING:
validateCastAsString(output, conditional);
break;
-
+
+ case LIST:
+ validateNamedList(output, conditional);
+ break;
+
default: //always unconditional (because unsupported operation)
//handle common issue of transformencode
if( getOpCode()==ParameterizedBuiltinFunctionOp.TRANSFORMENCODE )
@@ -761,6 +768,16 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
output.setValueType(ValueType.STRING);
output.setDimensions(0, 0);
}
+
+ private void validateNamedList(DataIdentifier output, boolean conditional) {
+ HashMap<String, Expression> varParams = getVarParams();
+
+ // set output characteristics
+ output.setDataType(DataType.LIST);
+ output.setValueType(ValueType.UNKNOWN);
+ output.setDimensions(varParams.size(), 1);
+ output.setBlockDimensions(-1, -1);
+ }
private void checkDataType( String fname, String pname, DataType dt, boolean conditional ) {
Expression data = getVarParam(pname);
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index 9e81f08..16db227 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -186,6 +186,7 @@ public class CPInstructionParser extends InstructionParser
String2CPInstructionType.put( "list", CPType.BuiltinNary);
// Parameterized Builtin Functions
+ String2CPInstructionType.put( "nvlist", CPType.ParameterizedBuiltin);
String2CPInstructionType.put( "cdf", CPType.ParameterizedBuiltin);
String2CPInstructionType.put( "invcdf", CPType.ParameterizedBuiltin);
String2CPInstructionType.put( "groupedagg", CPType.ParameterizedBuiltin);
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java
index 59b2d48..4890439 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListIndexingCPInstruction.java
@@ -21,9 +21,9 @@ package org.apache.sysml.runtime.instructions.cp;
import org.apache.sysml.lops.LeftIndex;
import org.apache.sysml.lops.RightIndex;
+import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.util.IndexRange;
public final class ListIndexingCPInstruction extends IndexingCPInstruction {
@@ -40,15 +40,22 @@ public final class ListIndexingCPInstruction extends IndexingCPInstruction {
@Override
public void processInstruction(ExecutionContext ec) {
String opcode = getOpcode();
- IndexRange ixrange = getIndexRange(ec);
+ ScalarObject rl = ec.getScalarInput(rowLower.getName(), rowLower.getValueType(), rowLower.isLiteral());
+ ScalarObject ru = ec.getScalarInput(rowUpper.getName(), rowUpper.getValueType(), rowUpper.isLiteral());
//right indexing
if( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) {
ListObject list = (ListObject) ec.getVariable(input1.getName());
//execute right indexing operation and set output
- ec.setVariable(output.getName(),
- list.slice((int)ixrange.rowStart, (int)ixrange.rowEnd));
+ if( rl.getValueType()==ValueType.STRING || ru.getValueType()==ValueType.STRING ) {
+ ec.setVariable(output.getName(),
+ list.slice(rl.getStringValue(), ru.getStringValue()));
+ }
+ else {
+ ec.setVariable(output.getName(),
+ list.slice((int)rl.getLongValue()-1, (int)ru.getLongValue()-1));
+ }
}
//left indexing
else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
index f773847..0fdb1e8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
@@ -72,17 +72,36 @@ public class ListObject extends Data
public Data slice(String name) {
//check for existing named list
if( _names == null )
- throw new DMLRuntimeException("Invalid lookup by name in unnamed list.");
+ throw new DMLRuntimeException("Invalid lookup by name"
+ + " in unnamed list: "+name+".");
//find position and check for existing entry
int pos = _names.indexOf(name);
if( pos <= 0 || pos >= _data.size() )
- throw new DMLRuntimeException("List lookup return no entry for name='"+name+"'");
+ throw new DMLRuntimeException("List lookup returned no entry for name='"+name+"'");
//return existing entry
return slice(pos);
}
+ public ListObject slice(String name1, String name2) {
+ //check for existing named list
+ if( _names == null )
+ throw new DMLRuntimeException("Invalid lookup by name"
+ + " in unnamed list: "+name1+", "+name2+".");
+
+ //find position and check for existing entry
+ int pos1 = _names.indexOf(name1);
+ int pos2 = _names.indexOf(name2);
+ if( pos1 <= 0 || pos1 >= _data.size() )
+ throw new DMLRuntimeException("List lookup returned no entry for name='"+name1+"'");
+ if( pos2 <= 0 || pos2 >= _data.size() )
+ throw new DMLRuntimeException("List lookup returned no entry for name='"+name2+"'");
+
+ //return list object
+ return slice(pos1, pos2);
+ }
+
@Override
public String getDebugName() {
return toString();
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index 06aeef3..8fac54c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -19,8 +19,12 @@
package org.apache.sysml.runtime.instructions.cp;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.stream.Collectors;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
@@ -55,9 +59,9 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
private static final String TOSTRING_SEPARATOR = " ";
private static final String TOSTRING_LINESEPARATOR = "\n";
- protected final HashMap<String, String> params;
+ protected final LinkedHashMap<String, String> params;
- protected ParameterizedBuiltinCPInstruction(Operator op, HashMap<String, String> paramsMap, CPOperand out,
+ protected ParameterizedBuiltinCPInstruction(Operator op, LinkedHashMap<String, String> paramsMap, CPOperand out,
String opcode, String istr) {
super(CPType.ParameterizedBuiltin, op, null, null, out, opcode, istr);
params = paramsMap;
@@ -71,9 +75,9 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
return getParameterMap().get(key);
}
- public static HashMap<String, String> constructParameterMap(String[] params) {
+ public static LinkedHashMap<String, String> constructParameterMap(String[] params) {
// process all elements in "params" except first(opcode) and last(output)
- HashMap<String,String> paramMap = new HashMap<>();
+ LinkedHashMap<String,String> paramMap = new LinkedHashMap<>();
// all parameters are of form <name=value>
String[] parts;
@@ -93,7 +97,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
CPOperand out = new CPOperand( parts[parts.length-1] );
// process remaining parts and build a hash map
- HashMap<String,String> paramsMap = constructParameterMap(parts);
+ LinkedHashMap<String,String> paramsMap = constructParameterMap(parts);
// determine the appropriate value function
ValueFunction func = null;
@@ -136,11 +140,9 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
else if ( opcode.equals("transformapply")
|| opcode.equals("transformdecode")
|| opcode.equals("transformcolmap")
- || opcode.equals("transformmeta"))
- {
- return new ParameterizedBuiltinCPInstruction(null, paramsMap, out, opcode, str);
- }
- else if ( opcode.equals("toString"))
+ || opcode.equals("transformmeta")
+ || opcode.equals("toString")
+ || opcode.equals("nvlist") )
{
return new ParameterizedBuiltinCPInstruction(null, paramsMap, out, opcode, str);
}
@@ -337,6 +339,20 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
ec.releaseCacheableData(getParam("target"));
ec.setScalarOutput(output.getName(), new StringObject(out));
}
+ else if( opcode.equals("nvlist") ) {
+ //obtain all input data objects and names in insertion order
+ List<Data> data = params.values().stream().map(d -> ec.containsVariable(d) ?
+ ec.getVariable(d) : new StringObject(d)).collect(Collectors.toList());
+ List<String> names = new ArrayList<>(params.keySet());
+
+ //create list object over all inputs
+ ListObject list = new ListObject(data, names);
+
+ //disable cleanup of individual objects and store cleanup state
+ list.setStatus(ec.pinVariables(new ArrayList<>(params.values())));
+
+ ec.setVariable(output.getName(), list);
+ }
else {
throw new DMLRuntimeException("Unknown opcode : " + opcode);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
index d2ca6f5..6acef43 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
@@ -94,7 +94,7 @@ public class ScalarBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction {
//obtain all input data objects, incl handling of literals
List<Data> data = Arrays.stream(inputs)
.map(in -> ec.getVariable(in)).collect(Collectors.toList());
-
+
//create list object over all inputs
ListObject list = new ListObject(data);
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
index b831032..a282b09 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
@@ -55,15 +55,15 @@ public class ListAndStructTest extends AutomatedTestBase
runListStructTest(TEST_NAME1, true);
}
-// @Test
-// public void testListNamed() {
-// runListStructTest(TEST_NAME2, false);
-// }
-//
-// @Test
-// public void testListNamedRewrites() {
-// runListStructTest(TEST_NAME2, true);
-// }
+ @Test
+ public void testListNamed() {
+ runListStructTest(TEST_NAME2, false);
+ }
+
+ @Test
+ public void testListNamedRewrites() {
+ runListStructTest(TEST_NAME2, true);
+ }
private void runListStructTest(String testname, boolean rewrites)
{
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/test/scripts/functions/misc/ListNamed.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListNamed.R b/src/test/scripts/functions/misc/ListNamed.R
new file mode 100644
index 0000000..abf01be
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListNamed.R
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(a=A, b=B, c=c, d=D, e=matrix(5, 3, 3), f=6);
+
+for( i in 1:length(X) ) {
+ tmp = X[i]
+ if( !exists("tmp") )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X[['e']])));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[1], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/systemml/blob/f9020a16/src/test/scripts/functions/misc/ListNamed.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListNamed.dml b/src/test/scripts/functions/misc/ListNamed.dml
new file mode 100644
index 0000000..5b9b4e8
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListNamed.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(a=A, b=B, c=c, d=D, e=matrix(5, 3, 3), f=6);
+
+for( i in 1:length(X) ) {
+ tmp = X[i];
+ if( !exists(tmp) )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X['e'])));
+
+write(R, $1);
[4/4] systemml git commit: [MINOR] Fix uaggouterchain compilation
(output data types)
Posted by mb...@apache.org.
[MINOR] Fix uaggouterchain compilation (output data types)
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/6f2c885e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/6f2c885e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/6f2c885e
Branch: refs/heads/master
Commit: 6f2c885e8aad480349e039fcd0390feb341b3639
Parents: f9020a1
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu May 10 12:27:58 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu May 10 12:28:57 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/hops/AggUnaryOp.java | 8 +++---
.../cp/UaggOuterChainCPInstruction.java | 27 +++++---------------
.../binary/matrix/UaggOuterChainTest.java | 3 +--
3 files changed, 12 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/6f2c885e/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index 136d2d6..d3e0570 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -191,14 +191,14 @@ public class AggUnaryOp extends Hop implements MultiThreadedHop
if (getDataType() == DataType.SCALAR) {
UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR),
- getDataType(), getValueType());
+ getDataType(), getValueType());
unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
setLineNumbers(unary1);
- setLops(unary1);
+ agg1 = unary1;
}
- }
- else { //general case
+ }
+ else { //general case
int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
agg1 = new PartialAggregate(input.constructLops(),
HopsAgg2Lops.get(_op), HopsDirection2Lops.get(_direction), getDataType(),getValueType(), et, k);
http://git-wip-us.apache.org/repos/asf/systemml/blob/6f2c885e/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
index 908e5bd..e6dd403 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/UaggOuterChainCPInstruction.java
@@ -74,7 +74,7 @@ public class UaggOuterChainCPInstruction extends UnaryCPInstruction {
boolean rightCached = (_uaggOp.indexFn instanceof ReduceCol || _uaggOp.indexFn instanceof ReduceAll
|| !LibMatrixOuterAgg.isSupportedUaggOp(_uaggOp, _bOp));
- MatrixBlock mbLeft = null, mbRight = null, mbOut = null;
+ MatrixBlock mbLeft = null, mbRight = null, mbOut = null;
//get the main data input
if( rightCached ) {
mbLeft = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
@@ -94,26 +94,13 @@ public class UaggOuterChainCPInstruction extends UnaryCPInstruction {
if( _uaggOp.aggOp.correctionExists )
mbOut.dropLastRowsOrColumns(_uaggOp.aggOp.correctionLocation);
- String output_name = output.getName();
- //final aggregation if required
- if(_uaggOp.indexFn instanceof ReduceAll ) //RC AGG (output is scalar)
- {
- //create and set output scalar
- ScalarObject ret = null;
- switch( output.getValueType() ) {
- case DOUBLE: ret = new DoubleObject(mbOut.quickGetValue(0, 0)); break;
-
- default:
- throw new DMLRuntimeException("Invalid output value type: "+output.getValueType());
- }
- ec.setScalarOutput(output_name, ret);
+ if(_uaggOp.indexFn instanceof ReduceAll ) { //RC AGG (output is scalar)
+ ec.setMatrixOutput(output.getName(), new MatrixBlock(
+ mbOut.quickGetValue(0, 0)), getExtendedOpcode());
}
- else //R/C AGG (output is rdd)
- {
- //Additional memory requirement to convert from dense to sparse can be leveraged from released memory needed for input data above.
+ else { //R/C AGG (output is rdd)
mbOut.examSparsity();
- ec.setMatrixOutput(output_name, mbOut, getExtendedOpcode());
+ ec.setMatrixOutput(output.getName(), mbOut, getExtendedOpcode());
}
-
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/6f2c885e/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
index 04a00c9..e031b53 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/UaggOuterChainTest.java
@@ -44,7 +44,6 @@ import org.apache.sysml.utils.Statistics;
*/
public class UaggOuterChainTest extends AutomatedTestBase
{
-
private final static String TEST_NAME1 = "UaggOuterChain";
private final static String TEST_DIR = "functions/binary/matrix/";
private final static String TEST_CLASS_DIR = TEST_DIR + UaggOuterChainTest.class.getSimpleName() + "/";
@@ -1318,7 +1317,7 @@ public class UaggOuterChainTest extends AutomatedTestBase
loadTestConfiguration(config, TEST_CACHE_DIR);
- String HOME = SCRIPT_DIR + TEST_DIR;
+ String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + TEST_NAME + suffix + strSumTypeSuffix + ".dml";
programArgs = new String[]{"-stats", "-explain","-args",
input("A"), input("B"), output("C")};
[2/4] systemml git commit: [MINOR] Simplification 1x1 matrix
initialization with scalar
Posted by mb...@apache.org.
[MINOR] Simplification 1x1 matrix initialization with scalar
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/fa6394cc
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/fa6394cc
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/fa6394cc
Branch: refs/heads/master
Commit: fa6394cc6981e80b262549ab86bd535124c3212b
Parents: d6e2ae6
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 9 20:47:30 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu May 10 12:28:39 2018 -0700
----------------------------------------------------------------------
.../runtime/instructions/cp/QuaternaryCPInstruction.java | 10 +++-------
.../runtime/instructions/cp/VariableCPInstruction.java | 3 +--
.../runtime/instructions/mr/QuaternaryInstruction.java | 3 +--
.../org/apache/sysml/runtime/matrix/data/MatrixBlock.java | 8 +++-----
.../functions/misc/UnivariateStatsBasicTest.java | 3 +--
5 files changed, 9 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/fa6394cc/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
index 85d2422..e9ed9a9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuaternaryCPInstruction.java
@@ -106,14 +106,10 @@ public class QuaternaryCPInstruction extends ComputationCPInstruction {
MatrixBlock matBlock3 = ec.getMatrixInput(input3.getName(), getExtendedOpcode());
MatrixBlock matBlock4 = null;
if( qop.hasFourInputs() ) {
- if (input4.getDataType() == DataType.SCALAR) {
- matBlock4 = new MatrixBlock(1, 1, false);
- final double eps = ec.getScalarInput(input4.getName(), input4.getValueType(), input4.isLiteral()).getDoubleValue();
- matBlock4.quickSetValue(0, 0, eps);
- }
- else {
+ if (input4.getDataType() == DataType.SCALAR)
+ matBlock4 = new MatrixBlock(ec.getScalarInput(input4).getDoubleValue());
+ else
matBlock4 = ec.getMatrixInput(input4.getName(), getExtendedOpcode());
- }
}
//core execute
http://git-wip-us.apache.org/repos/asf/systemml/blob/fa6394cc/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index 6d30346..5786e87 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -581,8 +581,7 @@ public class VariableCPInstruction extends CPInstruction {
else if( getInput1().getDataType()==DataType.SCALAR ) {
ScalarObject scalarInput = ec.getScalarInput(
getInput1().getName(), getInput1().getValueType(), getInput1().isLiteral());
- MatrixBlock out = new MatrixBlock(1,1,false);
- out.quickSetValue(0, 0, scalarInput.getDoubleValue());
+ MatrixBlock out = new MatrixBlock(scalarInput.getDoubleValue());
ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
}
else if( getInput1().getDataType()==DataType.LIST ) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/fa6394cc/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
index bf7dfdf..da8bd12 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/QuaternaryInstruction.java
@@ -309,9 +309,8 @@ public class QuaternaryInstruction extends MRInstruction implements IDistributed
IndexedMatrixValue iWij = (_input4 != -1) ? cachedValues.getFirst(_input4) : null;
MatrixValue Wij = (iWij!=null) ? iWij.getValue() : null;
if (null == Wij && qop.hasFourInputs()) {
- MatrixBlock mb = new MatrixBlock(1, 1, false);
String[] parts = InstructionUtils.getInstructionParts(instString);
- mb.quickSetValue(0, 0, Double.valueOf(parts[4]));
+ MatrixBlock mb = new MatrixBlock(Double.valueOf(parts[4]));
Wij = mb;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/fa6394cc/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index ae48c1b..61a1086 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -5235,14 +5235,12 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
LibMatrixMult.matrixMultWSigmoid(X, U, V, R, qop.wtype2, k);
else
LibMatrixMult.matrixMultWSigmoid(X, U, V, R, qop.wtype2);
- }
+ }
else if( qop.wtype3 != null ){ //wdivmm
//note: for wdivmm-minus X and W interchanged because W always present
MatrixBlock W = qop.wtype3.hasFourInputs() ? checkType(wm) : null;
- if( qop.getScalar() != 0 ) {
- W = new MatrixBlock(1, 1, false);
- W.quickSetValue(0, 0, qop.getScalar());
- }
+ if( qop.getScalar() != 0 )
+ W = new MatrixBlock(qop.getScalar());
if( k > 1 )
LibMatrixMult.matrixMultWDivMM(X, U, V, W, R, qop.wtype3, k);
else
http://git-wip-us.apache.org/repos/asf/systemml/blob/fa6394cc/src/test/java/org/apache/sysml/test/integration/functions/misc/UnivariateStatsBasicTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/UnivariateStatsBasicTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/UnivariateStatsBasicTest.java
index 849b293..525a92d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/UnivariateStatsBasicTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/UnivariateStatsBasicTest.java
@@ -77,8 +77,7 @@ public class UnivariateStatsBasicTest extends AutomatedTestBase
runTest(true, false, null, -1);
//write input types
- MatrixBlock mb = new MatrixBlock(1,1,false);
- mb.quickSetValue(0, 0, 1);
+ MatrixBlock mb = new MatrixBlock(1d);
MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo)
.writeMatrixToHDFS(mb, input("uni-types.csv"), 1, 1, 1, 1, 1);
MapReduceTool.writeMetaDataFile(input("uni-types.csv.mtd"), ValueType.DOUBLE,