You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/09/06 04:18:43 UTC
[1/2] systemml git commit: [MINOR] Rename rangeReIndex to rightIndex
and related cleanups
Repository: systemml
Updated Branches:
refs/heads/master d7f765331 -> d5f20b43b
[MINOR] Rename rangeReIndex to rightIndex and related cleanups
This patch renames the rangeReIndex to rightIndex to make it consistent
with the existing leftIndex. Furthermore, this also centralizes these
left/right indexing opcodes and modifies their usage accordingly.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/e60a4c29
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/e60a4c29
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/e60a4c29
Branch: refs/heads/master
Commit: e60a4c29dd1e6f3ccdcee8cae27628b75c2d4742
Parents: d7f7653
Author: Matthias Boehm <mb...@gmail.com>
Authored: Tue Sep 5 12:29:29 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Tue Sep 5 21:18:09 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/hops/IndexingOp.java | 8 +-
.../org/apache/sysml/hops/LeftIndexingOp.java | 4 +-
.../hops/cost/CostEstimatorStaticRuntime.java | 14 +-
.../java/org/apache/sysml/lops/LeftIndex.java | 6 +-
src/main/java/org/apache/sysml/lops/Lop.java | 20 +-
.../apache/sysml/lops/RangeBasedReIndex.java | 191 -------------------
.../java/org/apache/sysml/lops/RightIndex.java | 188 ++++++++++++++++++
.../instructions/CPInstructionParser.java | 6 +-
.../instructions/GPUInstructionParser.java | 4 +-
.../instructions/MRInstructionParser.java | 11 +-
.../instructions/SPInstructionParser.java | 8 +-
.../cp/FrameIndexingCPInstruction.java | 6 +-
.../instructions/cp/IndexingCPInstruction.java | 47 ++---
.../cp/MatrixIndexingCPInstruction.java | 6 +-
.../cpfile/MatrixIndexingCPFileInstruction.java | 29 ++-
.../gpu/MatrixIndexingGPUInstruction.java | 28 ++-
.../runtime/instructions/mr/MRInstruction.java | 2 +-
.../mr/RangeBasedReIndexInstruction.java | 7 +-
.../spark/FrameIndexingSPInstruction.java | 17 +-
.../spark/IndexingSPInstruction.java | 18 +-
.../spark/MatrixIndexingSPInstruction.java | 18 +-
.../runtime/matrix/data/LibMatrixCUDA.java | 2 +-
.../sysml/runtime/matrix/data/MatrixBlock.java | 2 +-
.../functions/codegen/RowAggTmplTest.java | 5 +-
.../recompile/RemoveEmptyRecompileTest.java | 12 +-
25 files changed, 312 insertions(+), 347 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/hops/IndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java b/src/main/java/org/apache/sysml/hops/IndexingOp.java
index 6a3ddf4..2463c7d 100644
--- a/src/main/java/org/apache/sysml/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java
@@ -27,7 +27,7 @@ import org.apache.sysml.lops.Data;
import org.apache.sysml.lops.Group;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopsException;
-import org.apache.sysml.lops.RangeBasedReIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -134,7 +134,7 @@ public class IndexingOp extends Hop
input._dim1, input._dim2, _dim1, _dim2);
Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
- RangeBasedReIndex reindex = new RangeBasedReIndex(
+ RightIndex reindex = new RightIndex(
input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(),
getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy,
getDataType(), getValueType(), et);
@@ -170,7 +170,7 @@ public class IndexingOp extends Hop
SparkAggType.NONE : SparkAggType.MULTI_BLOCK;
Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
- RangeBasedReIndex reindex = new RangeBasedReIndex(
+ RightIndex reindex = new RightIndex(
input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(),
getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy,
getDataType(), getValueType(), aggtype, et);
@@ -182,7 +182,7 @@ public class IndexingOp extends Hop
else //CP or GPU
{
Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
- RangeBasedReIndex reindex = new RangeBasedReIndex(
+ RightIndex reindex = new RightIndex(
input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(),
getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy,
getDataType(), getValueType(), et);
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
index 02e7753..f3c6614 100644
--- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
@@ -26,7 +26,7 @@ import org.apache.sysml.lops.LeftIndex;
import org.apache.sysml.lops.LeftIndex.LixCacheType;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopsException;
-import org.apache.sysml.lops.RangeBasedReIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.UnaryCP;
import org.apache.sysml.lops.ZeroOut;
import org.apache.sysml.lops.LopProperties.ExecType;
@@ -144,7 +144,7 @@ public class LeftIndexingOp extends Hop
rightInput = getInput().get(1).constructLops();
- RangeBasedReIndex reindex = new RangeBasedReIndex(
+ RightIndex reindex = new RightIndex(
rightInput, top, bottom,
left, right, nrow, ncol,
getDataType(), getValueType(), et, true);
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
index bb3ce3c..7cdde47 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
@@ -24,8 +24,10 @@ import java.util.HashSet;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.LeftIndex;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.MapMult;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
import org.apache.sysml.lops.compile.JobType;
@@ -1047,12 +1049,12 @@ public class CostEstimatorStaticRuntime extends CostEstimator
}
return 0;
- case MatrixIndexing: //opcodes: rangeReIndex, leftIndex
- if( optype.equals("leftIndex") ){
+ case MatrixIndexing: //opcodes: rightIndex, leftIndex
+ if( optype.equals(LeftIndex.OPCODE) ){
return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n)
+ 2 * DEFAULT_NFLOP_CP * ((rightSparse)? d2m*d2n*d2s : d2m*d2n );
}
- else if( optype.equals("rangeReIndex") ){
+ else if( optype.equals(RightIndex.OPCODE) ){
return DEFAULT_NFLOP_CP * ((leftSparse)? d2m*d2n*d2s : d2m*d2n );
}
return 0;
@@ -1200,11 +1202,11 @@ public class CostEstimatorStaticRuntime extends CostEstimator
//String2MRInstructionType.put( "valuepick" , MRINSTRUCTION_TYPE.PickByCount); // for quantile()
//String2MRInstructionType.put( "rangepick" , MRINSTRUCTION_TYPE.PickByCount); // for interQuantile()
- case RangeReIndex: //opcodes: rangeReIndex, rangeReIndexForLeft
+ case RightIndex: //opcodes: rightIndex, rightIndexForLeft
//TODO: requires category consolidation
- if( optype.equals("rangeReIndex") )
+ if( optype.equals(RightIndex.OPCODE) )
return DEFAULT_NFLOP_CP * ((leftSparse)? d2m*d2n*d2s : d2m*d2n );
- else //rangeReIndexForLeft
+ else //rightIndexForLeft
return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n)
+ DEFAULT_NFLOP_CP * ((rightSparse)? d2m*d2n*d2s : d2m*d2n );
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/LeftIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/LeftIndex.java b/src/main/java/org/apache/sysml/lops/LeftIndex.java
index 7c6117a..e841059 100644
--- a/src/main/java/org/apache/sysml/lops/LeftIndex.java
+++ b/src/main/java/org/apache/sysml/lops/LeftIndex.java
@@ -34,6 +34,8 @@ public class LeftIndex extends Lop
NONE
}
+ public static final String OPCODE = "leftIndex";
+
private LixCacheType _type;
public LeftIndex(
@@ -104,7 +106,7 @@ public class LeftIndex extends Lop
if( _type != LixCacheType.NONE )
return "mapLeftIndex";
else
- return "leftIndex";
+ return OPCODE;
}
@Override
@@ -153,6 +155,6 @@ public class LeftIndex extends Lop
@Override
public String toString() {
- return "leftIndex";
+ return getOpcode();
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java
index 33fbccb..238e329 100644
--- a/src/main/java/org/apache/sysml/lops/Lop.java
+++ b/src/main/java/org/apache/sysml/lops/Lop.java
@@ -38,27 +38,27 @@ public abstract class Lop
{
public enum Type {
- Data, DataGen, //CP/MR read/write/datagen
- ReBlock, CSVReBlock, //MR reblock operations
+ Data, DataGen, //CP/MR read/write/datagen
+ ReBlock, CSVReBlock, //MR reblock operations
MMCJ, MMRJ, MMTSJ, PMMJ, MapMult, MapMultChain, //MR matrix multiplications
UnaryCP, UNARY, BinaryCP, Binary, Ternary, //CP/MR unary/binary/ternary
- RangeReIndex, LeftIndex, ZeroOut, //CP/MR indexing
- Aggregate, PartialAggregate, //CP/MR aggregation
- BinUaggChain, UaggOuterChain, //CP/MR aggregation
+ RightIndex, LeftIndex, ZeroOut, //CP/MR indexing
+ Aggregate, PartialAggregate, //CP/MR aggregation
+ BinUaggChain, UaggOuterChain, //CP/MR aggregation
TernaryAggregate, //CP ternary-binary aggregates
- Grouping, //MR grouping
+ Grouping, //MR grouping
Append, //CP/MR append (column append)
CombineUnary, CombineBinary, CombineTernary, //MR combine (stitch together)
CentralMoment, CoVariance, GroupedAgg, GroupedAggM,
Transform, DataPartition, RepMat, //CP/MR reorganization, partitioning, replication
ParameterizedBuiltin, //CP/MR parameterized ops (name/value)
- FunctionCallCP, FunctionCallCPSingle, //CP function calls
+ FunctionCallCP, FunctionCallCPSingle, //CP function calls
CumulativePartialAggregate, CumulativeSplitAggregate, CumulativeOffsetBinary, //MR cumsum/cumprod/cummin/cummax
WeightedSquaredLoss, WeightedSigmoid, WeightedDivMM, WeightedCeMM, WeightedUMM,
SortKeys, PickValues,
- Checkpoint, //Spark persist into storage level
- PlusMult, MinusMult, //CP
- SpoofFused, //CP/SP generated fused operator
+ Checkpoint, //Spark persist into storage level
+ PlusMult, MinusMult, //CP
+ SpoofFused, //CP/SP generated fused operator
/** CP operation on a variable number of operands */
MULTIPLE_CP
};
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java b/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java
deleted file mode 100644
index 401dfbd..0000000
--- a/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.lops;
-
-import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
-import org.apache.sysml.lops.LopProperties.ExecLocation;
-import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.lops.compile.JobType;
-import org.apache.sysml.parser.Expression.DataType;
-import org.apache.sysml.parser.Expression.ValueType;
-
-
-public class RangeBasedReIndex extends Lop
-{
-
- private boolean forLeftIndexing = false;
-
- //optional attribute for spark exec type
- private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, ExecType et, boolean forleft)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, forleft);
- }
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
- }
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, SparkAggType aggtype, ExecType et)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- _aggtype = aggtype;
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
- }
-
- private void init(Lop inputMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop leftMatrixRowDim,
- Lop leftMatrixColDim, DataType dt, ValueType vt, ExecType et, boolean forleft)
- {
- addInput(inputMatrix);
- addInput(rowL);
- addInput(rowU);
- addInput(colL);
- addInput(colU);
- addInput(leftMatrixRowDim);
- addInput(leftMatrixColDim);
-
- inputMatrix.addOutput(this);
- rowL.addOutput(this);
- rowU.addOutput(this);
- colL.addOutput(this);
- colU.addOutput(this);
- leftMatrixRowDim.addOutput(this);
- leftMatrixColDim.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
-
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
-
- forLeftIndexing=forleft;
- }
-
- private String getOpcode() {
- if(forLeftIndexing)
- return "rangeReIndexForLeft";
- else
- return "rangeReIndex";
- }
-
- @Override
- public String getInstructions(String input, String rowl, String rowu, String coll, String colu, String leftRowDim, String leftColDim, String output)
- throws LopsException {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
-
- // rowl, rowu
- sb.append( getInputs().get(1).prepScalarInputOperand(rowl));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(rowu));
- sb.append( OPERAND_DELIMITOR );
-
- // coll, colu
- sb.append( getInputs().get(3).prepScalarInputOperand(coll));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(4).prepScalarInputOperand(colu));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( output );
- sb.append( DATATYPE_PREFIX );
- sb.append( getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getValueType() );
-
- if(getExecType() == ExecType.MR) {
- // following fields are added only when this lop is executed in MR (both for left & right indexing)
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(5).prepScalarInputOperand(leftRowDim));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(6).prepScalarInputOperand(leftColDim));
- }
-
- //in case of spark, we also compile the optional aggregate flag into the instruction.
- if( getExecType() == ExecType.SPARK ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _aggtype );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int input_index6, int input_index7, int output_index)
- throws LopsException {
- /*
- * Example: B = A[row_l:row_u, col_l:col_u]
- * A - input matrix (input_index1)
- * row_l - lower bound in row dimension
- * row_u - upper bound in row dimension
- * col_l - lower bound in column dimension
- * col_u - upper bound in column dimension
- *
- * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5,6,7)
- * will be equal to -1. They should be ignored and the scalar value labels must
- * be derived from input lops.
- */
- String rowl = getInputs().get(1).prepScalarLabel();
- String rowu = getInputs().get(2).prepScalarLabel();
- String coll = getInputs().get(3).prepScalarLabel();
- String colu = getInputs().get(4).prepScalarLabel();
-
- String left_nrow = getInputs().get(5).prepScalarLabel();
- String left_ncol = getInputs().get(6).prepScalarLabel();
-
- return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, left_nrow, left_ncol, Integer.toString(output_index));
- }
-
- @Override
- public String toString() {
- if(forLeftIndexing)
- return "rangeReIndexForLeft";
- else
- return "rangeReIndex";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/RightIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/RightIndex.java b/src/main/java/org/apache/sysml/lops/RightIndex.java
new file mode 100644
index 0000000..812425a
--- /dev/null
+++ b/src/main/java/org/apache/sysml/lops/RightIndex.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.lops;
+
+import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+
+
+public class RightIndex extends Lop
+{
+ public static final String OPCODE = "rightIndex";
+
+ private boolean forLeftIndexing = false;
+
+ //optional attribute for spark exec type
+ private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
+
+ public RightIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
+ DataType dt, ValueType vt, ExecType et, boolean forleft)
+ throws LopsException
+ {
+ super(Lop.Type.RightIndex, dt, vt);
+ init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, forleft);
+ }
+
+ public RightIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
+ DataType dt, ValueType vt, ExecType et)
+ throws LopsException
+ {
+ super(Lop.Type.RightIndex, dt, vt);
+ init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
+ }
+
+ public RightIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
+ DataType dt, ValueType vt, SparkAggType aggtype, ExecType et)
+ throws LopsException
+ {
+ super(Lop.Type.RightIndex, dt, vt);
+ _aggtype = aggtype;
+ init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
+ }
+
+ private void init(Lop inputMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop leftMatrixRowDim,
+ Lop leftMatrixColDim, DataType dt, ValueType vt, ExecType et, boolean forleft)
+ {
+ addInput(inputMatrix);
+ addInput(rowL);
+ addInput(rowU);
+ addInput(colL);
+ addInput(colU);
+ addInput(leftMatrixRowDim);
+ addInput(leftMatrixColDim);
+
+ inputMatrix.addOutput(this);
+ rowL.addOutput(this);
+ rowU.addOutput(this);
+ colL.addOutput(this);
+ colU.addOutput(this);
+ leftMatrixRowDim.addOutput(this);
+ leftMatrixColDim.addOutput(this);
+
+ boolean breaksAlignment = true;
+ boolean aligner = false;
+ boolean definesMRJob = false;
+
+ if ( et == ExecType.MR ) {
+ lps.addCompatibility(JobType.GMR);
+ lps.addCompatibility(JobType.DATAGEN);
+ lps.addCompatibility(JobType.MMCJ);
+ lps.addCompatibility(JobType.MMRJ);
+ lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
+ }
+ else {
+ lps.addCompatibility(JobType.INVALID);
+ lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
+ }
+
+ forLeftIndexing=forleft;
+ }
+
+ private String getOpcode() {
+ if(forLeftIndexing)
+ return OPCODE+"ForLeft";
+ else
+ return OPCODE;
+ }
+
+ @Override
+ public String getInstructions(String input, String rowl, String rowu, String coll, String colu, String leftRowDim, String leftColDim, String output)
+ throws LopsException {
+ StringBuilder sb = new StringBuilder();
+ sb.append( getExecType() );
+ sb.append( OPERAND_DELIMITOR );
+ sb.append( getOpcode() );
+ sb.append( OPERAND_DELIMITOR );
+
+ sb.append( getInputs().get(0).prepInputOperand(input));
+ sb.append( OPERAND_DELIMITOR );
+
+ // rowl, rowu
+ sb.append( getInputs().get(1).prepScalarInputOperand(rowl));
+ sb.append( OPERAND_DELIMITOR );
+ sb.append( getInputs().get(2).prepScalarInputOperand(rowu));
+ sb.append( OPERAND_DELIMITOR );
+
+ // coll, colu
+ sb.append( getInputs().get(3).prepScalarInputOperand(coll));
+ sb.append( OPERAND_DELIMITOR );
+ sb.append( getInputs().get(4).prepScalarInputOperand(colu));
+ sb.append( OPERAND_DELIMITOR );
+
+ sb.append( output );
+ sb.append( DATATYPE_PREFIX );
+ sb.append( getDataType() );
+ sb.append( VALUETYPE_PREFIX );
+ sb.append( getValueType() );
+
+ if(getExecType() == ExecType.MR) {
+ // following fields are added only when this lop is executed in MR (both for left & right indexing)
+ sb.append( OPERAND_DELIMITOR );
+
+ sb.append( getInputs().get(5).prepScalarInputOperand(leftRowDim));
+ sb.append( OPERAND_DELIMITOR );
+ sb.append( getInputs().get(6).prepScalarInputOperand(leftColDim));
+ }
+
+ //in case of spark, we also compile the optional aggregate flag into the instruction.
+ if( getExecType() == ExecType.SPARK ) {
+ sb.append( OPERAND_DELIMITOR );
+ sb.append( _aggtype );
+ }
+
+ return sb.toString();
+ }
+
+ @Override
+ public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int input_index6, int input_index7, int output_index)
+ throws LopsException {
+ /*
+ * Example: B = A[row_l:row_u, col_l:col_u]
+ * A - input matrix (input_index1)
+ * row_l - lower bound in row dimension
+ * row_u - upper bound in row dimension
+ * col_l - lower bound in column dimension
+ * col_u - upper bound in column dimension
+ *
+ * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5,6,7)
+ * will be equal to -1. They should be ignored and the scalar value labels must
+ * be derived from input lops.
+ */
+ String rowl = getInputs().get(1).prepScalarLabel();
+ String rowu = getInputs().get(2).prepScalarLabel();
+ String coll = getInputs().get(3).prepScalarLabel();
+ String colu = getInputs().get(4).prepScalarLabel();
+
+ String left_nrow = getInputs().get(5).prepScalarLabel();
+ String left_ncol = getInputs().get(6).prepScalarLabel();
+
+ return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, left_nrow, left_ncol, Integer.toString(output_index));
+ }
+
+ @Override
+ public String toString() {
+ return getOpcode();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index f1c9d3e..2f77710 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -24,7 +24,9 @@ import java.util.HashMap;
import org.apache.sysml.lops.Append;
import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.LeftIndex;
import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.UnaryCP;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.cp.AggregateBinaryCPInstruction;
@@ -257,8 +259,8 @@ public class CPInstructionParser extends InstructionParser
String2CPInstructionType.put( "qpick" , CPINSTRUCTION_TYPE.QPick);
- String2CPInstructionType.put( "rangeReIndex", CPINSTRUCTION_TYPE.MatrixIndexing);
- String2CPInstructionType.put( "leftIndex" , CPINSTRUCTION_TYPE.MatrixIndexing);
+ String2CPInstructionType.put( RightIndex.OPCODE, CPINSTRUCTION_TYPE.MatrixIndexing);
+ String2CPInstructionType.put( LeftIndex.OPCODE, CPINSTRUCTION_TYPE.MatrixIndexing);
String2CPInstructionType.put( "tsmm" , CPINSTRUCTION_TYPE.MMTSJ);
String2CPInstructionType.put( "pmm" , CPINSTRUCTION_TYPE.PMMJ);
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java
index 8abfc69..6f1ed91 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java
@@ -20,6 +20,7 @@ package org.apache.sysml.runtime.instructions;
import java.util.HashMap;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.gpu.AggregateBinaryGPUInstruction;
import org.apache.sysml.runtime.instructions.gpu.ArithmeticBinaryGPUInstruction;
@@ -131,8 +132,7 @@ public class GPUInstructionParser extends InstructionParser
String2GPUInstructionType.put( ">=" , GPUINSTRUCTION_TYPE.RelationalBinary);
// Indexing
- // right indexing: output = X[1:3, 4:5]
- String2GPUInstructionType.put( "rangeReIndex", GPUINSTRUCTION_TYPE.MatrixIndexing);
+ String2GPUInstructionType.put( RightIndex.OPCODE, GPUINSTRUCTION_TYPE.MatrixIndexing);
}
public static GPUInstruction parseSingleInstruction (String str )
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
index 5ced2ab..afc9d35 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java
@@ -25,6 +25,7 @@ import org.apache.sysml.lops.BinaryM;
import org.apache.sysml.lops.DataGen;
import org.apache.sysml.lops.MapMult;
import org.apache.sysml.lops.MapMultChain;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.WeightedCrossEntropy;
import org.apache.sysml.lops.WeightedCrossEntropyR;
import org.apache.sysml.lops.WeightedDivMM;
@@ -258,10 +259,10 @@ public class MRInstructionParser extends InstructionParser
String2MRInstructionType.put( "groupedagg" , MRINSTRUCTION_TYPE.GroupedAggregate);
String2MRInstructionType.put( "mapgroupedagg" , MRINSTRUCTION_TYPE.MapGroupedAggregate);
- //rangereindexing
- String2MRInstructionType.put( "rangeReIndex" , MRINSTRUCTION_TYPE.RangeReIndex);
- String2MRInstructionType.put( "rangeReIndexForLeft" , MRINSTRUCTION_TYPE.RangeReIndex);
- String2MRInstructionType.put( "zeroOut" , MRINSTRUCTION_TYPE.ZeroOut);
+ //right indexing
+ String2MRInstructionType.put( RightIndex.OPCODE , MRINSTRUCTION_TYPE.RightIndex);
+ String2MRInstructionType.put( RightIndex.OPCODE+"ForLeft" , MRINSTRUCTION_TYPE.RightIndex);
+ String2MRInstructionType.put( "zeroOut" , MRINSTRUCTION_TYPE.ZeroOut);
//append
String2MRInstructionType.put( "mappend" , MRINSTRUCTION_TYPE.Append);
@@ -413,7 +414,7 @@ public class MRInstructionParser extends InstructionParser
case MapGroupedAggregate:
return GroupedAggregateMInstruction.parseInstruction(str);
- case RangeReIndex:
+ case RightIndex:
return RangeBasedReIndexInstruction.parseInstruction(str);
case ZeroOut:
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
index a2b0ef1..2ebb2e1 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
@@ -24,6 +24,8 @@ import java.util.HashMap;
import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.Compression;
import org.apache.sysml.lops.DataGen;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.WeightedCrossEntropy;
import org.apache.sysml.lops.WeightedCrossEntropyR;
import org.apache.sysml.lops.WeightedDivMM;
@@ -141,9 +143,9 @@ public class SPInstructionParser extends InstructionParser
String2SPInstructionType.put( "maxpooling", SPINSTRUCTION_TYPE.Convolution);
String2SPInstructionType.put( "relu_maxpooling", SPINSTRUCTION_TYPE.Convolution);
- String2SPInstructionType.put( "rangeReIndex" , SPINSTRUCTION_TYPE.MatrixIndexing);
- String2SPInstructionType.put( "leftIndex" , SPINSTRUCTION_TYPE.MatrixIndexing);
- String2SPInstructionType.put( "mapLeftIndex" , SPINSTRUCTION_TYPE.MatrixIndexing);
+ String2SPInstructionType.put( RightIndex.OPCODE, SPINSTRUCTION_TYPE.MatrixIndexing);
+ String2SPInstructionType.put( LeftIndex.OPCODE, SPINSTRUCTION_TYPE.MatrixIndexing);
+ String2SPInstructionType.put( "mapLeftIndex" , SPINSTRUCTION_TYPE.MatrixIndexing);
// Reorg Instruction Opcodes (repositioning of existing values)
String2SPInstructionType.put( "r'" , SPINSTRUCTION_TYPE.Reorg);
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java
index fa45b9c..d585148 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java
@@ -19,6 +19,8 @@
package org.apache.sysml.runtime.instructions.cp;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
@@ -46,7 +48,7 @@ public final class FrameIndexingCPInstruction extends IndexingCPInstruction {
IndexRange ixrange = getIndexRange(ec);
//right indexing
- if( opcode.equalsIgnoreCase("rangeReIndex") )
+ if( opcode.equalsIgnoreCase(RightIndex.OPCODE) )
{
//execute right indexing operation
FrameBlock in = ec.getFrameInput(input1.getName());
@@ -59,7 +61,7 @@ public final class FrameIndexingCPInstruction extends IndexingCPInstruction {
ec.setFrameOutput(output.getName(), out);
}
//left indexing
- else if ( opcode.equalsIgnoreCase("leftIndex"))
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE))
{
FrameBlock lin = ec.getFrameInput(input1.getName());
FrameBlock out = null;
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
index 46f3f05..e3cb100 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java
@@ -19,6 +19,8 @@
package org.apache.sysml.runtime.instructions.cp;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
@@ -62,22 +64,15 @@ public abstract class IndexingCPInstruction extends UnaryCPInstruction {
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
String opcode = parts[0];
- if ( opcode.equalsIgnoreCase("rangeReIndex") ) {
+ if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) {
if ( parts.length == 7 ) {
- // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
CPOperand in, rl, ru, cl, cu, out;
- in = new CPOperand();
- rl = new CPOperand();
- ru = new CPOperand();
- cl = new CPOperand();
- cu = new CPOperand();
- out = new CPOperand();
- in.split(parts[1]);
- rl.split(parts[2]);
- ru.split(parts[3]);
- cl.split(parts[4]);
- cu.split(parts[5]);
- out.split(parts[6]);
+ in = new CPOperand(parts[1]);
+ rl = new CPOperand(parts[2]);
+ ru = new CPOperand(parts[3]);
+ cl = new CPOperand(parts[4]);
+ cu = new CPOperand(parts[5]);
+ out = new CPOperand(parts[6]);
if( in.getDataType()==DataType.MATRIX )
return new MatrixIndexingCPInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str);
else if (in.getDataType() == DataType.FRAME)
@@ -89,24 +84,16 @@ public abstract class IndexingCPInstruction extends UnaryCPInstruction {
throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
}
}
- else if ( opcode.equalsIgnoreCase("leftIndex")) {
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) {
if ( parts.length == 8 ) {
- // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7
CPOperand lhsInput, rhsInput, rl, ru, cl, cu, out;
- lhsInput = new CPOperand();
- rhsInput = new CPOperand();
- rl = new CPOperand();
- ru = new CPOperand();
- cl = new CPOperand();
- cu = new CPOperand();
- out = new CPOperand();
- lhsInput.split(parts[1]);
- rhsInput.split(parts[2]);
- rl.split(parts[3]);
- ru.split(parts[4]);
- cl.split(parts[5]);
- cu.split(parts[6]);
- out.split(parts[7]);
+ lhsInput = new CPOperand(parts[1]);
+ rhsInput = new CPOperand(parts[2]);
+ rl = new CPOperand(parts[3]);
+ ru = new CPOperand(parts[4]);
+ cl = new CPOperand(parts[5]);
+ cu = new CPOperand(parts[6]);
+ out = new CPOperand(parts[7]);
if( lhsInput.getDataType()==DataType.MATRIX )
return new MatrixIndexingCPInstruction(new SimpleOperator(null), lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, str);
else if (lhsInput.getDataType() == DataType.FRAME)
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
index 942f898..85c2a3d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java
@@ -20,6 +20,8 @@
package org.apache.sysml.runtime.instructions.cp;
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -54,7 +56,7 @@ public final class MatrixIndexingCPInstruction extends IndexingCPInstruction {
MatrixObject mo = ec.getMatrixObject(input1.getName());
//right indexing
- if( opcode.equalsIgnoreCase("rangeReIndex") )
+ if( opcode.equalsIgnoreCase(RightIndex.OPCODE) )
{
MatrixBlock resultBlock = null;
@@ -78,7 +80,7 @@ public final class MatrixIndexingCPInstruction extends IndexingCPInstruction {
ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode());
}
//left indexing
- else if ( opcode.equalsIgnoreCase("leftIndex"))
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE))
{
UpdateType updateType = mo.getUpdateType();
if(DMLScript.STATISTICS)
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
index 33f3aed..acfd4d4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
@@ -19,6 +19,8 @@
package org.apache.sysml.runtime.instructions.cpfile;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
@@ -55,29 +57,22 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
String opcode = parts[0];
- if ( opcode.equalsIgnoreCase("rangeReIndex") ) {
+ if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) {
if ( parts.length == 7 ) {
- // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
CPOperand in, rl, ru, cl, cu, out;
- in = new CPOperand();
- rl = new CPOperand();
- ru = new CPOperand();
- cl = new CPOperand();
- cu = new CPOperand();
- out = new CPOperand();
- in.split(parts[1]);
- rl.split(parts[2]);
- ru.split(parts[3]);
- cl.split(parts[4]);
- cu.split(parts[5]);
- out.split(parts[6]);
+ in = new CPOperand(parts[1]);
+ rl = new CPOperand(parts[2]);
+ ru = new CPOperand(parts[3]);
+ cl = new CPOperand(parts[4]);
+ cu = new CPOperand(parts[5]);
+ out = new CPOperand(parts[6]);
return new MatrixIndexingCPFileInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str);
}
else {
throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
}
}
- else if ( parts[0].equalsIgnoreCase("leftIndex"))
+ else if ( parts[0].equalsIgnoreCase(LeftIndex.OPCODE))
{
throw new DMLRuntimeException("Invalid opcode while parsing a MatrixIndexingCPFileInstruction: " + str);
}
@@ -95,7 +90,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
IndexRange ixrange = getIndexRange(ec).add(1);
MatrixObject mo = ec.getMatrixObject(input1.getName());
- if( mo.isPartitioned() && opcode.equalsIgnoreCase("rangeReIndex") )
+ if( mo.isPartitioned() && opcode.equalsIgnoreCase(RightIndex.OPCODE) )
{
MatrixFormatMetaData meta = (MatrixFormatMetaData)mo.getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
@@ -125,7 +120,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
mcNew = new MatrixCharacteristics( mc.getRows(), mo.getPartitionSize(), mc.getRowsPerBlock(), mc.getColsPerBlock() );
break;
default:
- throw new DMLRuntimeException("Unsupported partition format for CP_FILE rangeReIndex: "+ mo.getPartitionFormat());
+ throw new DMLRuntimeException("Unsupported partition format for CP_FILE "+RightIndex.OPCODE+": "+ mo.getPartitionFormat());
}
MatrixFormatMetaData metaNew = new MatrixFormatMetaData(mcNew,meta.getOutputInfo(),meta.getInputInfo());
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java
index ba3ccb5..9b29d4a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java
@@ -18,6 +18,8 @@
*/
package org.apache.sysml.runtime.instructions.gpu;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
@@ -65,22 +67,15 @@ public class MatrixIndexingGPUInstruction extends GPUInstruction {
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
String opcode = parts[0];
- if ( opcode.equalsIgnoreCase("rangeReIndex") ) {
+ if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) {
if ( parts.length == 7 ) {
- // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
CPOperand in, rl, ru, cl, cu, out;
- in = new CPOperand();
- rl = new CPOperand();
- ru = new CPOperand();
- cl = new CPOperand();
- cu = new CPOperand();
- out = new CPOperand();
- in.split(parts[1]);
- rl.split(parts[2]);
- ru.split(parts[3]);
- cl.split(parts[4]);
- cu.split(parts[5]);
- out.split(parts[6]);
+ in = new CPOperand(parts[1]);
+ rl = new CPOperand(parts[2]);
+ ru = new CPOperand(parts[3]);
+ cl = new CPOperand(parts[4]);
+ cu = new CPOperand(parts[5]);
+ out = new CPOperand(parts[6]);
if( in.getDataType()==DataType.MATRIX )
return new MatrixIndexingGPUInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str);
else
@@ -90,9 +85,8 @@ public class MatrixIndexingGPUInstruction extends GPUInstruction {
throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
}
}
- else if ( opcode.equalsIgnoreCase("leftIndex")) {
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) {
if ( parts.length == 8 ) {
- // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7
CPOperand lhsInput, rhsInput, rl, ru, cl, cu, out;
lhsInput = new CPOperand();
rhsInput = new CPOperand();
@@ -129,7 +123,7 @@ public class MatrixIndexingGPUInstruction extends GPUInstruction {
String opcode = getOpcode();
IndexRange ixrange = getIndexRange(ec);
- if ( opcode.equalsIgnoreCase("rangeReIndex") ) {
+ if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) {
MatrixObject mat1 = getMatrixInputForGPUInstruction(ec, input1.getName());
LibMatrixCUDA.sliceOperations(ec, ec.getGPUContext(0), getExtendedOpcode(), mat1, ixrange, output.getName());
ec.releaseMatrixInputForGPUInstruction(input1.getName());
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
index 7d05010..0fd160a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java
@@ -32,7 +32,7 @@ public abstract class MRInstruction extends Instruction {
public enum MRINSTRUCTION_TYPE {
INVALID, Append, Aggregate, ArithmeticBinary, ArithmeticBinary2, AggregateBinary, AggregateUnary, Rand,
Seq, CSVReblock, CSVWrite, Reblock, Reorg, Replicate, Unary, CombineBinary, CombineUnary, CombineTernary,
- PickByCount, Partition, Ternary, Quaternary, CM_N_COV, MapGroupedAggregate, GroupedAggregate, RangeReIndex,
+ PickByCount, Partition, Ternary, Quaternary, CM_N_COV, MapGroupedAggregate, GroupedAggregate, RightIndex,
ZeroOut, MMTSJ, PMMJ, MatrixReshape, ParameterizedBuiltin, Sort, MapMultChain, CumsumAggregate, CumsumSplit,
CumsumOffset, BinUaggChain, UaggOuterChain, RemoveEmpty
};
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
index 79d15ec..b7d73ea 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
@@ -21,6 +21,7 @@ package org.apache.sysml.runtime.instructions.mr;
import java.util.ArrayList;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -42,7 +43,7 @@ public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase {
private RangeBasedReIndexInstruction(Operator op, byte in, byte out, IndexRange rng, boolean forleft, long rlen,
long clen, String istr) {
super(op, in, out);
- mrtype = MRINSTRUCTION_TYPE.RangeReIndex;
+ mrtype = MRINSTRUCTION_TYPE.RightIndex;
instString = istr;
_ixrange = rng;
_forLeft = forleft;
@@ -66,9 +67,9 @@ public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase {
String opcode = parts[0];
boolean forLeft = false;
- if(opcode.equalsIgnoreCase("rangeReIndexForLeft"))
+ if(opcode.equalsIgnoreCase(RightIndex.OPCODE+"ForLeft"))
forLeft=true;
- else if(!opcode.equalsIgnoreCase("rangeReIndex"))
+ else if(!opcode.equalsIgnoreCase(RightIndex.OPCODE))
throw new DMLRuntimeException("Unknown opcode while parsing a Select: " + str);
byte in = Byte.parseByte(parts[1]);
IndexRange rng=new IndexRange(UtilFunctions.parseToLong(parts[2]),
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
index 71178c3..71f17a0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
@@ -29,6 +29,8 @@ import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
@@ -48,16 +50,7 @@ import org.apache.sysml.runtime.util.IndexRange;
import org.apache.sysml.runtime.util.UtilFunctions;
/**
- * This class implements the frame indexing functionality inside Spark.
- * Example instructions:
- * rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
- * input=mVar1, output=mVar6,
- * bounds = (Var2,Var3,Var4,Var5)
- * rowindex_lower: Var2, rowindex_upper: Var3
- * colindex_lower: Var4, colindex_upper: Var5
- * leftIndex:mVar1:mVar2:Var3:Var4:Var5:Var6:mVar7
- * triggered by "mVar1[Var3:Var4, Var5:Var6] = mVar2"
- * the result is stored in mVar7
+ * This class implements the frame indexing functionality inside Spark.
*
*/
public class FrameIndexingSPInstruction extends IndexingSPInstruction {
@@ -87,7 +80,7 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction {
IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
//right indexing
- if( opcode.equalsIgnoreCase("rangeReIndex") )
+ if( opcode.equalsIgnoreCase(RightIndex.OPCODE) )
{
//update and check output dimensions
MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
@@ -116,7 +109,7 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction {
sec.getFrameObject(input1.getName()).getSchema((int)cl, (int)cu));
}
//left indexing
- else if ( opcode.equalsIgnoreCase("leftIndex") || opcode.equalsIgnoreCase("mapLeftIndex"))
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex"))
{
JavaPairRDD<Long,FrameBlock> in1 = sec.getFrameBinaryBlockRDDHandleForVariable( input1.getName() );
PartitionedBroadcast<FrameBlock> broadcastIn2 = null;
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java
index 0932cdd..38558d7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java
@@ -20,6 +20,8 @@
package org.apache.sysml.runtime.instructions.spark;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.LeftIndex.LixCacheType;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -30,16 +32,6 @@ import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
/**
* This class implements the matrix indexing functionality inside Spark.
- * Example instructions:
- * rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
- * input=mVar1, output=mVar6,
- * bounds = (Var2,Var3,Var4,Var5)
- * rowindex_lower: Var2, rowindex_upper: Var3
- * colindex_lower: Var4, colindex_upper: Var5
- * leftIndex:mVar1:mVar2:Var3:Var4:Var5:Var6:mVar7
- * triggered by "mVar1[Var3:Var4, Var5:Var6] = mVar2"
- * the result is stored in mVar7
- *
*/
public abstract class IndexingSPInstruction extends UnarySPInstruction {
protected CPOperand rowLower, rowUpper, colLower, colUpper;
@@ -71,9 +63,8 @@ public abstract class IndexingSPInstruction extends UnarySPInstruction {
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
String opcode = parts[0];
- if ( opcode.equalsIgnoreCase("rangeReIndex") ) {
+ if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) {
if ( parts.length == 8 ) {
- // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
CPOperand in = new CPOperand(parts[1]);
CPOperand rl = new CPOperand(parts[2]);
CPOperand ru = new CPOperand(parts[3]);
@@ -90,9 +81,8 @@ public abstract class IndexingSPInstruction extends UnarySPInstruction {
throw new DMLRuntimeException("Invalid number of operands in instruction: " + str);
}
}
- else if ( opcode.equalsIgnoreCase("leftIndex") || opcode.equalsIgnoreCase("mapLeftIndex")) {
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex")) {
if ( parts.length == 9 ) {
- // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7
CPOperand lhsInput = new CPOperand(parts[1]);
CPOperand rhsInput = new CPOperand(parts[2]);
CPOperand rl = new CPOperand(parts[3]);
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
index b4a0aeb..540ac74 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
@@ -37,6 +37,8 @@ import scala.reflect.ClassManifestFactory;
import scala.runtime.AbstractFunction1;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.lops.LeftIndex.LixCacheType;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -59,17 +61,7 @@ import org.apache.sysml.runtime.util.IndexRange;
import org.apache.sysml.runtime.util.UtilFunctions;
/**
- * This class implements the matrix indexing functionality inside CP.
- * Example instructions:
- * rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6
- * input=mVar1, output=mVar6,
- * bounds = (Var2,Var3,Var4,Var5)
- * rowindex_lower: Var2, rowindex_upper: Var3
- * colindex_lower: Var4, colindex_upper: Var5
- * leftIndex:mVar1:mVar2:Var3:Var4:Var5:Var6:mVar7
- * triggered by "mVar1[Var3:Var4, Var5:Var6] = mVar2"
- * the result is stored in mVar7
- *
+ * This class implements the matrix indexing functionality inside CP.
*/
public class MatrixIndexingSPInstruction extends IndexingSPInstruction {
private final LixCacheType _type;
@@ -101,7 +93,7 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction {
IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
//right indexing
- if( opcode.equalsIgnoreCase("rangeReIndex") )
+ if( opcode.equalsIgnoreCase(RightIndex.OPCODE) )
{
//update and check output dimensions
MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
@@ -127,7 +119,7 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction {
}
}
//left indexing
- else if ( opcode.equalsIgnoreCase("leftIndex") || opcode.equalsIgnoreCase("mapLeftIndex"))
+ else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex"))
{
String rddVar = (_type==LixCacheType.LEFT) ? input2.getName() : input1.getName();
String bcVar = (_type==LixCacheType.LEFT) ? input1.getName() : input2.getName();
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
index 4be5c2d..09ffe9f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
@@ -3087,7 +3087,7 @@ public class LibMatrixCUDA {
//********************************************************************/
/**
- * Method to perform rangeReIndex operation for a given lower and upper bounds in row and column dimensions.
+ * Method to perform rightIndex operation for a given lower and upper bounds in row and column dimensions.
*
* @param ec current execution context
* @param gCtx current gpu context
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 145eb97..d041fc4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -3781,7 +3781,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
}
/**
- * Method to perform rangeReIndex operation for a given lower and upper bounds in row and column dimensions.
+ * Method to perform rightIndex operation for a given lower and upper bounds in row and column dimensions.
* Extracted submatrix is returned as "result". Note: This operation is now 0-based.
*
* @param rl row lower
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java
index 6d25130..3ecfd6b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java
@@ -28,6 +28,7 @@ import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -573,13 +574,13 @@ public class RowAggTmplTest extends AutomatedTestBase
if( testname.equals(TEST_NAME15) )
Assert.assertTrue(!heavyHittersContainsSubString("uark+"));
if( testname.equals(TEST_NAME17) )
- Assert.assertTrue(!heavyHittersContainsSubString("rangeReIndex"));
+ Assert.assertTrue(!heavyHittersContainsSubString(RightIndex.OPCODE));
if( testname.equals(TEST_NAME28) )
Assert.assertTrue(!heavyHittersContainsSubString("spoofRA", 2)
&& !heavyHittersContainsSubString("sp_spoofRA", 2));
if( testname.equals(TEST_NAME30) )
Assert.assertTrue(!heavyHittersContainsSubString("spoofRA", 2)
- && !heavyHittersContainsSubString("rangeReIndex"));
+ && !heavyHittersContainsSubString(RightIndex.OPCODE));
}
finally {
rtplatform = platformOld;
http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
index 630c774..c1bac4e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java
@@ -25,6 +25,8 @@ import org.junit.Assert;
import org.junit.Test;
import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LeftIndex;
+import org.apache.sysml.lops.RightIndex;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -305,9 +307,9 @@ public class RemoveEmptyRecompileTest extends AutomatedTestBase
{
switch(type){
//for sum, literal replacement of unary aggregates applies
- case SUM: return "rlit";//return "uak+";
+ case SUM: return "rlit";//return "uak+";
- case ROUND: return "round";
+ case ROUND: return "round";
case TRANSPOSE: return "r'";
case MULT_LEFT:
case MULT_RIGHT: return "*";
@@ -316,9 +318,9 @@ public class RemoveEmptyRecompileTest extends AutomatedTestBase
case MINUS_LEFT:
case MINUS_RIGHT: return "-";
case MM_LEFT:
- case MM_RIGHT: return "ba+*";
- case RIX: return "rangeReIndex";
- case LIX: return "leftIndex";
+ case MM_RIGHT: return "ba+*";
+ case RIX: return RightIndex.OPCODE;
+ case LIX: return LeftIndex.OPCODE;
}
return null;
[2/2] systemml git commit: [SYSTEMML-1889] Robustness parfor/hop
memory budgets in spark exec mode
Posted by mb...@apache.org.
[SYSTEMML-1889] Robustness parfor/hop memory budgets in spark exec mode
Recently, we encountered robustness issues of parfor memory budgets in
spark execution mode (remote memory set to Infinity). This patch
improves the robustness of analyzing the spark cluster configuration,
and the parfor-specific infrastructure analysis. The encountered issue
was likely due to an invalid default parallelism (less than the number
of executors), leading to MEM/(dpar/#exec)=INF.
Furthermore, this patch also updates the internal defaults of min and
max storage memory fractions to the defaults used in Spark 2.1 and 2.2.
Additionally, parfor now also reports the entire Spark cluster config
when log=DEBUG, without unnecessarily creating the Spark context.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d5f20b43
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d5f20b43
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d5f20b43
Branch: refs/heads/master
Commit: d5f20b43b9e74e73747ee65f11f9767577b001e8
Parents: e60a4c2
Author: Matthias Boehm <mb...@gmail.com>
Authored: Tue Sep 5 17:57:06 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Tue Sep 5 21:18:10 2017 -0700
----------------------------------------------------------------------
.../context/SparkExecutionContext.java | 28 +++++++++++---------
.../parfor/opt/OptimizerRuleBased.java | 12 ++++++---
2 files changed, 24 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d5f20b43/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index d1ff7d8..e0352b0 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -1382,7 +1382,7 @@ public class SparkExecutionContext extends ExecutionContext
{
//broadcasts are stored in mem-and-disk in data space, this config
//defines the fraction of data space to be used as broadcast budget
- private static final double BROADCAST_DATA_FRACTION = 0.3;
+ private static final double BROADCAST_DATA_FRACTION = 0.35;
//forward private config from Spark's UnifiedMemoryManager.scala (>1.6)
private static final long RESERVED_SYSTEM_MEMORY_BYTES = 300 * 1024 * 1024;
@@ -1430,7 +1430,7 @@ public class SparkExecutionContext extends ExecutionContext
//always get the current num executors on refresh because this might
//change if not all executors are initially allocated and it is plan-relevant
int numExec = _numExecutors;
- if( refresh && !_confOnly ) {
+ if( (refresh && !_confOnly) || isSparkContextCreated() ) {
JavaSparkContext jsc = getSparkContextStatic();
numExec = Math.max(jsc.sc().getExecutorMemoryStatus().size() - 1, 1);
}
@@ -1452,14 +1452,15 @@ public class SparkExecutionContext extends ExecutionContext
//always get the current default parallelism on refresh because this might
//change if not all executors are initially allocated and it is plan-relevant
- return ( refresh && !_confOnly ) ?
+ int par = ( (refresh && !_confOnly) || isSparkContextCreated() ) ?
getSparkContextStatic().defaultParallelism() : _defaultPar;
+ return Math.max(par, 1); //robustness min parallelism
}
public void analyzeSparkConfiguationLegacy(SparkConf conf) {
//ensure allocated spark conf
SparkConf sconf = (conf == null) ? createSystemMLSparkConf() : conf;
-
+
//parse absolute executor memory
_memExecutor = UtilFunctions.parseMemorySize(
sconf.get("spark.executor.memory", "1g"));
@@ -1477,7 +1478,7 @@ public class SparkExecutionContext extends ExecutionContext
public void analyzeSparkConfiguation(SparkConf conf) {
//ensure allocated spark conf
SparkConf sconf = (conf == null) ? createSystemMLSparkConf() : conf;
-
+
//parse absolute executor memory, incl fixed cut off
_memExecutor = UtilFunctions.parseMemorySize(
sconf.get("spark.executor.memory", "1g"))
@@ -1485,14 +1486,17 @@ public class SparkExecutionContext extends ExecutionContext
//get data and shuffle memory ratios (defaults not specified in job conf)
_memDataMinFrac = sconf.getDouble("spark.memory.storageFraction", 0.5); //default 50%
- _memDataMaxFrac = sconf.getDouble("spark.memory.fraction", 0.75); //default 75%
- _memBroadcastFrac = _memDataMaxFrac * BROADCAST_DATA_FRACTION; //default 22.5%
-
+ _memDataMaxFrac = sconf.getDouble("spark.memory.fraction", 0.6); //default 60%
+ _memBroadcastFrac = _memDataMaxFrac * BROADCAST_DATA_FRACTION; //default 21%
+
//analyze spark degree of parallelism
analyzeSparkParallelismConfiguation(sconf);
}
- private void analyzeSparkParallelismConfiguation(SparkConf sconf) {
+ private void analyzeSparkParallelismConfiguation(SparkConf conf) {
+ //ensure allocated spark conf
+ SparkConf sconf = (conf == null) ? createSystemMLSparkConf() : conf;
+
int numExecutors = sconf.getInt("spark.executor.instances", -1);
int numCoresPerExec = sconf.getInt("spark.executor.cores", -1);
int defaultPar = sconf.getInt("spark.default.parallelism", -1);
@@ -1532,14 +1536,14 @@ public class SparkExecutionContext extends ExecutionContext
@Override
public String toString() {
StringBuilder sb = new StringBuilder("SparkClusterConfig: \n");
- sb.append("-- legacyVersion = " + _legacyVersion + " ("+getSparkContextStatic().version()+")\n" );
+ sb.append("-- legacyVersion = " + _legacyVersion + " ("+getSparkVersionString()+")\n" );
sb.append("-- confOnly = " + _confOnly + "\n");
+ sb.append("-- numExecutors = " + _numExecutors + "\n");
+ sb.append("-- defaultPar = " + _defaultPar + "\n");
sb.append("-- memExecutor = " + _memExecutor + "\n");
sb.append("-- memDataMinFrac = " + _memDataMinFrac + "\n");
sb.append("-- memDataMaxFrac = " + _memDataMaxFrac + "\n");
sb.append("-- memBroadcastFrac = " + _memBroadcastFrac + "\n");
- sb.append("-- numExecutors = " + _numExecutors + "\n");
- sb.append("-- defaultPar = " + _defaultPar + "\n");
return sb.toString();
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d5f20b43/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index c429bfa..ec229d5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -213,9 +213,13 @@ public class OptimizerRuleBased extends Optimizer
_cost = est;
//debug and warnings output
- LOG.debug(getOptMode()+" OPT: Optimize w/ max_mem="+toMB(_lm)+"/"+toMB(_rm)+"/"+toMB(_rm2)+", max_k="+_lk+"/"+_rk+"/"+_rk2+")." );
- if( _rnk<=0 || _rk<=0 )
- LOG.warn(getOptMode()+" OPT: Optimize for inactive cluster (num_nodes="+_rnk+", num_map_slots="+_rk+")." );
+ if( LOG.isDebugEnabled() ) {
+ LOG.debug(getOptMode()+" OPT: Optimize w/ max_mem="+toMB(_lm)+"/"+toMB(_rm)+"/"+toMB(_rm2)+", max_k="+_lk+"/"+_rk+"/"+_rk2+")." );
+ if( OptimizerUtils.isSparkExecutionMode() )
+ LOG.debug(getOptMode()+" OPT: Optimize w/ "+SparkExecutionContext.getSparkClusterConfig().toString());
+ if( _rnk <= 0 || _rk <= 0 )
+ LOG.warn(getOptMode()+" OPT: Optimize for inactive cluster (num_nodes="+_rnk+", num_map_slots="+_rk+")." );
+ }
//ESTIMATE memory consumption
pn.setSerialParFor(); //for basic mem consumption
@@ -356,7 +360,7 @@ public class OptimizerRuleBased extends Optimizer
_rk2 = _rk; //equal map/reduce unless we find counter-examples
int cores = SparkExecutionContext.getDefaultParallelism(true)
/ SparkExecutionContext.getNumExecutors();
- int ccores = (int) Math.min(cores, _N);
+ int ccores = Math.max((int) Math.min(cores, _N), 1);
_rm = SparkExecutionContext.getBroadcastMemoryBudget() / ccores;
_rm2 = SparkExecutionContext.getBroadcastMemoryBudget() / ccores;
}