You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:45:57 UTC
[26/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java b/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java
deleted file mode 100644
index 68f4b74..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PartialAggregate.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform a partial aggregation. It was introduced to do some initial
- * aggregation operations on blocks in the mapper/reducer.
- */
-
-public class PartialAggregate extends Lop
-{
-
-
- public enum DirectionTypes {
- RowCol,
- Row,
- Col
- };
-
- public enum CorrectionLocationType {
- NONE,
- LASTROW,
- LASTCOLUMN,
- LASTTWOROWS,
- LASTTWOCOLUMNS,
- INVALID
- };
-
- private Aggregate.OperationTypes operation;
- private DirectionTypes direction;
- private boolean _dropCorr = false;
-
- //optional attribute for CP num threads
- private int _numThreads = -1;
-
- //optional attribute for spark exec type
- private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, ExecType.MR);
- }
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, et);
- }
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, ExecType et, int k)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, et);
- _numThreads = k;
- }
-
- public PartialAggregate( Lop input, Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, SparkAggType aggtype, ExecType et)
- throws LopsException
- {
- super(Lop.Type.PartialAggregate, dt, vt);
- init(input, op, direct, dt, vt, et);
- _aggtype = aggtype;
- }
-
- /**
- * Constructor to setup a partial aggregate operation.
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- private void init(Lop input,
- Aggregate.OperationTypes op,
- PartialAggregate.DirectionTypes direct, DataType dt, ValueType vt, ExecType et) {
- operation = op;
- direction = direct;
- this.addInput(input);
- input.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR )
- {
- /*
- * This lop CAN NOT be executed in PARTITION, SORT, STANDALONE MMCJ:
- * only in mapper.
- */
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- this.lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else //CP | SPARK
- {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- public void setDropCorrection()
- {
- _dropCorr = true;
- }
-
- public static CorrectionLocationType decodeCorrectionLocation(String loc)
- {
- return CorrectionLocationType.valueOf(loc);
- }
-
- /**
- * This method computes the location of "correction" terms in the output
- * produced by PartialAgg instruction.
- *
- * When computing the stable sum, "correction" refers to the compensation as
- * defined by the original Kahan algorithm. When computing the stable mean,
- * "correction" refers to two extra values (the running mean, count)
- * produced by each Mapper i.e., by each PartialAgg instruction.
- *
- * This method is invoked during hop-to-lop translation, while creating the
- * corresponding Aggregate lop
- *
- * Computed information is encoded in the PartialAgg instruction so that the
- * appropriate aggregate operator is used at runtime (see:
- * dml.runtime.matrix.operator.AggregateOperator.java and dml.runtime.matrix)
- */
- public CorrectionLocationType getCorrectionLocation()
- throws LopsException
- {
- return getCorrectionLocation(operation, direction);
- }
-
- public static CorrectionLocationType getCorrectionLocation(Aggregate.OperationTypes operation, DirectionTypes direction)
- throws LopsException
- {
- CorrectionLocationType loc;
-
- switch (operation) {
- case KahanSum:
- case KahanSumSq:
- case KahanTrace:
- switch (direction) {
- case Col:
- // colSums: corrections will be present as a last row in the
- // result
- loc = CorrectionLocationType.LASTROW;
- break;
- case Row:
- case RowCol:
- // rowSums, sum: corrections will be present as a last column in
- // the result
- loc = CorrectionLocationType.LASTCOLUMN;
- break;
- default:
- throw new LopsException("PartialAggregate.getCorrectionLocation() - "
- + "Unknown aggregate direction: " + direction);
- }
- break;
-
- case Mean:
- // Computation of stable mean requires each mapper to output both
- // the running mean as well as the count
- switch (direction) {
- case Col:
- // colMeans: last row is correction 2nd last is count
- loc = CorrectionLocationType.LASTTWOROWS;
- break;
- case Row:
- case RowCol:
- // rowMeans, mean: last column is correction 2nd last is count
- loc = CorrectionLocationType.LASTTWOCOLUMNS;
- break;
- default:
- throw new LopsException("PartialAggregate.getCorrectionLocation() - "
- + "Unknown aggregate direction: " + direction);
- }
- break;
-
- case MaxIndex:
- case MinIndex:
- loc = CorrectionLocationType.LASTCOLUMN;
- break;
-
- default:
- loc = CorrectionLocationType.NONE;
- }
- return loc;
- }
-
- public void setDimensionsBasedOnDirection(long dim1, long dim2,
- long rowsPerBlock, long colsPerBlock) throws LopsException
- {
- setDimensionsBasedOnDirection(this, dim1, dim2, rowsPerBlock, colsPerBlock, direction);
- }
-
- public static void setDimensionsBasedOnDirection(Lop lop, long dim1, long dim2,
- long rowsPerBlock, long colsPerBlock, DirectionTypes dir)
- throws LopsException
- {
- try {
- if (dir == DirectionTypes.Row)
- lop.outParams.setDimensions(dim1, 1, rowsPerBlock, colsPerBlock, -1);
- else if (dir == DirectionTypes.Col)
- lop.outParams.setDimensions(1, dim2, rowsPerBlock, colsPerBlock, -1);
- else if (dir == DirectionTypes.RowCol)
- lop.outParams.setDimensions(1, 1, rowsPerBlock, colsPerBlock, -1);
- else
- throw new LopsException("In PartialAggregate Lop, Unknown aggregate direction " + dir);
- } catch (HopsException e) {
- throw new LopsException("In PartialAggregate Lop, error setting dimensions based on direction", e);
- }
- }
-
- public String toString() {
- return "Partial Aggregate " + operation;
- }
-
- private String getOpcode() {
- return getOpcode(operation, direction);
- }
-
- /**
- * Instruction generation for for CP and Spark
- */
- @Override
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output) );
-
- //in case of spark, we also compile the optional aggregate flag into the instruction.
- if( getExecType() == ExecType.SPARK ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _aggtype );
- }
-
- //in case of cp, we also compile the number of threads into the instruction
- if( getExecType() == ExecType.CP ){
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( _dropCorr );
-
- return sb.toString();
- }
-
- /**
- *
- * @param op
- * @param dir
- * @return
- */
- public static String getOpcode(Aggregate.OperationTypes op, DirectionTypes dir)
- {
- switch( op )
- {
- case Sum: {
- if( dir == DirectionTypes.RowCol )
- return "ua+";
- else if( dir == DirectionTypes.Row )
- return "uar+";
- else if( dir == DirectionTypes.Col )
- return "uac+";
- break;
- }
-
- case Mean: {
- if( dir == DirectionTypes.RowCol )
- return "uamean";
- else if( dir == DirectionTypes.Row )
- return "uarmean";
- else if( dir == DirectionTypes.Col )
- return "uacmean";
- break;
- }
-
- case KahanSum: {
- // instructions that use kahanSum are similar to ua+,uar+,uac+
- // except that they also produce correction values along with partial
- // sums.
- if( dir == DirectionTypes.RowCol )
- return "uak+";
- else if( dir == DirectionTypes.Row )
- return "uark+";
- else if( dir == DirectionTypes.Col )
- return "uack+";
- break;
- }
-
- case KahanSumSq: {
- if( dir == DirectionTypes.RowCol )
- return "uasqk+";
- else if( dir == DirectionTypes.Row )
- return "uarsqk+";
- else if( dir == DirectionTypes.Col )
- return "uacsqk+";
- break;
- }
-
- case Product: {
- if( dir == DirectionTypes.RowCol )
- return "ua*";
- break;
- }
-
- case Max: {
- if( dir == DirectionTypes.RowCol )
- return "uamax";
- else if( dir == DirectionTypes.Row )
- return "uarmax";
- else if( dir == DirectionTypes.Col )
- return "uacmax";
- break;
- }
-
- case Min: {
- if( dir == DirectionTypes.RowCol )
- return "uamin";
- else if( dir == DirectionTypes.Row )
- return "uarmin";
- else if( dir == DirectionTypes.Col )
- return "uacmin";
- break;
- }
-
- case MaxIndex:{
- if( dir == DirectionTypes.Row )
- return "uarimax";
- break;
- }
-
- case MinIndex: {
- if( dir == DirectionTypes.Row )
- return "uarimin";
- break;
- }
-
- case Trace: {
- if( dir == DirectionTypes.RowCol)
- return "uatrace";
- break;
- }
-
- case KahanTrace: {
- if( dir == DirectionTypes.RowCol )
- return "uaktrace";
- break;
- }
- }
-
- //should never come here for normal compilation
- throw new UnsupportedOperationException("Instruction is not defined for PartialAggregate operation " + op);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PickByCount.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PickByCount.java b/src/main/java/com/ibm/bi/dml/lops/PickByCount.java
deleted file mode 100644
index f0cca24..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PickByCount.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class PickByCount extends Lop
-{
-
- public static final String OPCODE = "qpick";
-
- public enum OperationTypes {
- VALUEPICK,
- RANGEPICK,
- IQM,
- MEDIAN
- };
-
- private OperationTypes operation;
- private boolean inMemoryInput = false;
-
-
- /*
- * valuepick: first input is always a matrix, second input can either be a scalar or a matrix
- * rangepick: first input is always a matrix, second input is always a scalar
- */
- public PickByCount(Lop input1, Lop input2, DataType dt, ValueType vt, OperationTypes op) {
- this(input1, input2, dt, vt, op, ExecType.MR);
- }
-
- public PickByCount(Lop input1, Lop input2, DataType dt, ValueType vt, OperationTypes op, ExecType et) {
- super(Lop.Type.PickValues, dt, vt);
- init(input1, input2, op, et);
- }
-
- public PickByCount(Lop input1, Lop input2, DataType dt, ValueType vt, OperationTypes op, ExecType et, boolean inMemoryInput) {
- super(Lop.Type.PickValues, dt, vt);
- this.inMemoryInput = inMemoryInput;
- init(input1, input2, op, et);
- }
-
-
- private void init(Lop input1, Lop input2, OperationTypes op, ExecType et) {
- this.addInput(input1);
- input1.addOutput(this);
-
- if ( input2 != null ) {
- this.addInput(input2);
- input2.addOutput(this);
- }
-
- operation = op;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, et, ExecLocation.RecordReader, breaksAlignment, aligner, definesMRJob );
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return "Operation: " + operation;
- }
-
- public OperationTypes getOperationType() {
- return operation;
- }
-
- /*
- * This version of getInstruction() must be called only for valuepick (MR) and rangepick
- *
- * Example instances:
- * valupick:::0:DOUBLE:::1:DOUBLE:::2:DOUBLE
- * rangepick:::0:DOUBLE:::0.25:DOUBLE:::1:DOUBLE
- * rangepick:::0:DOUBLE:::Var3:DOUBLE:::1:DOUBLE
- */
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- throws LopsException
- {
- return getInstructions(""+input_index1, ""+input_index2, ""+output_index);
-
- }
-
- /*
- * This version of getInstructions() must be called only for valuepick (CP), IQM (CP)
- *
- * Example instances:
- * valuepick:::temp2:STRING:::0.25:DOUBLE:::Var1:DOUBLE
- * valuepick:::temp2:STRING:::Var1:DOUBLE:::Var2:DOUBLE
- */
- @Override
- public String getInstructions(String input1, String input2, String output) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- if(operation != OperationTypes.MEDIAN) {
- if ( getInputs().get(1).getDataType() == DataType.SCALAR )
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- else {
- sb.append( getInputs().get(1).prepInputOperand(input2));
- }
- sb.append( OPERAND_DELIMITOR );
- }
-
- sb.append( this.prepOutputOperand(output));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(operation);
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(inMemoryInput);
-
- return sb.toString();
- }
-
- /**
- * This version of getInstructions() is called for IQM, executing in CP
- *
- * Example instances:
- * iqm:::input:::output
- */
- @Override
- public String getInstructions(String input, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output) );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(operation);
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(inMemoryInput);
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java b/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java
deleted file mode 100644
index a36c0d3..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/RangeBasedReIndex.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class RangeBasedReIndex extends Lop
-{
-
- /**
- * Constructor to setup a RangeBasedReIndex operation.
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
-
- private boolean forLeftIndexing = false;
-
- //optional attribute for spark exec type
- private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, ExecType et, boolean forleft)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, forleft);
- }
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
- }
-
- public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim,
- DataType dt, ValueType vt, SparkAggType aggtype, ExecType et)
- throws LopsException
- {
- super(Lop.Type.RangeReIndex, dt, vt);
- _aggtype = aggtype;
- init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false);
- }
-
- private void init(Lop inputMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop leftMatrixRowDim,
- Lop leftMatrixColDim, DataType dt, ValueType vt, ExecType et, boolean forleft)
- {
- addInput(inputMatrix);
- addInput(rowL);
- addInput(rowU);
- addInput(colL);
- addInput(colU);
- addInput(leftMatrixRowDim);
- addInput(leftMatrixColDim);
-
- inputMatrix.addOutput(this);
- rowL.addOutput(this);
- rowU.addOutput(this);
- colL.addOutput(this);
- colU.addOutput(this);
- leftMatrixRowDim.addOutput(this);
- leftMatrixColDim.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
-
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
-
- forLeftIndexing=forleft;
- }
-
- private String getOpcode() {
- if(forLeftIndexing)
- return "rangeReIndexForLeft";
- else
- return "rangeReIndex";
- }
-
- @Override
- public String getInstructions(String input, String rowl, String rowu, String coll, String colu, String leftRowDim, String leftColDim, String output)
- throws LopsException {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
-
- // rowl, rowu
- sb.append( getInputs().get(1).prepScalarInputOperand(rowl));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(rowu));
- sb.append( OPERAND_DELIMITOR );
-
- // coll, colu
- sb.append( getInputs().get(3).prepScalarInputOperand(coll));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(4).prepScalarInputOperand(colu));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( output );
- sb.append( DATATYPE_PREFIX );
- sb.append( getDataType() );
- sb.append( VALUETYPE_PREFIX );
- sb.append( getValueType() );
-
- if(getExecType() == ExecType.MR) {
- // following fields are added only when this lop is executed in MR (both for left & right indexing)
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(5).prepScalarInputOperand(leftRowDim));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(6).prepScalarInputOperand(leftColDim));
- }
-
- //in case of spark, we also compile the optional aggregate flag into the instruction.
- if( getExecType() == ExecType.SPARK ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _aggtype );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int input_index6, int input_index7, int output_index)
- throws LopsException {
- /*
- * Example: B = A[row_l:row_u, col_l:col_u]
- * A - input matrix (input_index1)
- * row_l - lower bound in row dimension
- * row_u - upper bound in row dimension
- * col_l - lower bound in column dimension
- * col_u - upper bound in column dimension
- *
- * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5,6,7)
- * will be equal to -1. They should be ignored and the scalar value labels must
- * be derived from input lops.
- */
- String rowl = getInputs().get(1).prepScalarLabel();
- String rowu = getInputs().get(2).prepScalarLabel();
- String coll = getInputs().get(3).prepScalarLabel();
- String colu = getInputs().get(4).prepScalarLabel();
-
- String left_nrow = getInputs().get(5).prepScalarLabel();
- String left_ncol = getInputs().get(6).prepScalarLabel();
-
- return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, left_nrow, left_ncol, Integer.toString(output_index));
- }
-
- @Override
- public String toString() {
- if(forLeftIndexing)
- return "rangeReIndexForLeft";
- else
- return "rangeReIndex";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/ReBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/ReBlock.java b/src/main/java/com/ibm/bi/dml/lops/ReBlock.java
deleted file mode 100644
index a442fa6..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/ReBlock.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.OutputParameters.Format;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to perform reblock operation
- */
-public class ReBlock extends Lop
-{
-
- public static final String OPCODE = "rblk";
-
- private boolean _outputEmptyBlocks = true;
-
- /**
- * Constructor to perform a reblock operation.
- * @param input
- * @param op
- */
-
- private Long _rows_per_block;
- private Long _cols_per_block;
-
- public ReBlock(Lop input, Long rows_per_block, Long cols_per_block, DataType dt, ValueType vt, boolean outputEmptyBlocks, ExecType et) throws LopsException
- {
- super(Lop.Type.ReBlock, dt, vt);
- this.addInput(input);
- input.addOutput(this);
-
- _rows_per_block = rows_per_block;
- _cols_per_block = cols_per_block;
-
- _outputEmptyBlocks = outputEmptyBlocks;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
-
- lps.addCompatibility(JobType.REBLOCK);
-
- if(et == ExecType.MR)
- lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- else if(et == ExecType.SPARK)
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- else
- throw new LopsException("Incorrect execution type for Reblock:" + et);
- }
-
- @Override
- public String toString() {
-
- return "Reblock - rows per block = " + _rows_per_block + " cols per block " + _cols_per_block ;
- }
-
- @Override
- public String getInstructions(int input_index, int output_index) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append ( this.prepOutputOperand(output_index));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _rows_per_block );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cols_per_block );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(_outputEmptyBlocks);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output) throws LopsException {
- if(getExecType() != ExecType.SPARK) {
- throw new LopsException("The method getInstructions(String,String) for Reblock should be called only for Spark execution type");
- }
-
- if (this.getInputs().size() == 1) {
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "rblk" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
- sb.append( OPERAND_DELIMITOR );
- sb.append( _rows_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cols_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append(_outputEmptyBlocks);
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for Reblock operation");
- }
- }
-
- // This function is replicated in Dag.java
- @SuppressWarnings("unused")
- private Format getChildFormat(Lop node) throws LopsException {
-
- if(node.getOutputParameters().getFile_name() != null
- || node.getOutputParameters().getLabel() != null)
- {
- return node.getOutputParameters().getFormat();
- }
- else
- {
- // Reblock lop should always have a single child
- if(node.getInputs().size() > 1)
- throw new LopsException(this.printErrorLocation() + "Should only have one child! \n");
-
- /*
- * Return the format of the child node (i.e., input lop)
- * No need of recursion here.. because
- * 1) Reblock lop's input can either be DataLop or some intermediate computation
- * If it is Data then we just take its format (TEXT or BINARY)
- * If it is intermediate lop then it is always BINARY
- * since we assume that all intermediate computations will be in Binary format
- * 2) Note that Reblock job will never have any instructions in the mapper
- * => the input lop (if it is other than Data) is always executed in a different job
- */
- // return getChildFormat(node.getInputs().get(0));
- return node.getInputs().get(0).getOutputParameters().getFormat(); }
-
- }
-
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/RepMat.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/RepMat.java b/src/main/java/com/ibm/bi/dml/lops/RepMat.java
deleted file mode 100644
index b838539..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/RepMat.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class RepMat extends Lop
-{
-
- public static final String OPCODE = "rep";
-
- private boolean _repCols = true;
-
- /**
- * Constructor to setup a partial Matrix-Vector Multiplication
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public RepMat(Lop input1, Lop input2, boolean repCols, DataType dt, ValueType vt)
- throws LopsException
- {
- super(Lop.Type.RepMat, dt, vt);
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- _repCols = repCols;
-
- //setup MR parameters
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
-
-
- public String toString() {
- return "Operation = RepMat";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(OPCODE);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_repCols);
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/SortKeys.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/SortKeys.java b/src/main/java/com/ibm/bi/dml/lops/SortKeys.java
deleted file mode 100644
index 39b0ea7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/SortKeys.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-public class SortKeys extends Lop
-{
-
- public static final String OPCODE = "qsort"; //quantile sort
-
- public enum OperationTypes {
- WithWeights,
- WithoutWeights,
- Indexes,
- };
-
- private OperationTypes operation;
- private boolean descending = false;
-
- public OperationTypes getOpType() {
- return operation;
- }
-
- public SortKeys(Lop input, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input, null, op, ExecType.MR);
- }
-
- public SortKeys(Lop input, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input, null, op, et);
- }
-
- public SortKeys(Lop input, boolean desc, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input, null, op, et);
- descending = desc;
- }
-
- public SortKeys(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.SortKeys, dt, vt);
- init(input1, input2, op, et);
- }
-
- private void init(Lop input1, Lop input2, OperationTypes op, ExecType et) {
- this.addInput(input1);
- input1.addOutput(this);
-
- operation = op;
-
- if ( et == ExecType.MR ) {
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = true;
-
- lps.addCompatibility(JobType.SORT);
- this.lps.setProperties( inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- if(op != OperationTypes.Indexes)
- this.lps.setProducesIntermediateOutput(true);
- }
- else {
- // SortKeys can accept a optional second input only when executing in CP
- // Example: sorting with weights inside CP
- if ( input2 != null ) {
- this.addInput(input2);
- input2.addOutput(this);
- }
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, false, false, false);
- }
- }
-
-
- @Override
- public String toString() {
- return "Operation: SortKeys (" + operation + ")";
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- {
- return getInstructions(String.valueOf(input_index), String.valueOf(output_index));
- }
-
- @Override
- public String getInstructions(String input, String output)
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input));
- sb.append( OPERAND_DELIMITOR );
- sb.append ( this.prepOutputOperand(output));
-
- if( getExecType() == ExecType.MR ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( operation );
- sb.append( OPERAND_DELIMITOR );
- sb.append( descending );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- // This method is invoked in two cases:
- // 1) SortKeys (both weighted and unweighted) executes in MR
- // 2) Unweighted SortKeys executes in CP
- public static SortKeys constructSortByValueLop(Lop input1, OperationTypes op,
- DataType dt, ValueType vt, ExecType et) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.SortKeys ) {
- return (SortKeys)lop;
- }
- }
-
- SortKeys retVal = new SortKeys(input1, op, dt, vt, et);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
- // This method is invoked ONLY for the case of Weighted SortKeys executing in CP
- public static SortKeys constructSortByValueLop(Lop input1, Lop input2, OperationTypes op,
- DataType dt, ValueType vt, ExecType et) {
-
- HashSet<Lop> set1 = new HashSet<Lop>();
- set1.addAll(input1.getOutputs());
- // find intersection of input1.getOutputs() and input2.getOutputs();
- set1.retainAll(input2.getOutputs());
-
- for (Lop lop : set1) {
- if ( lop.type == Lop.Type.SortKeys ) {
- return (SortKeys)lop;
- }
- }
-
- SortKeys retVal = new SortKeys(input1, input2, op, dt, vt, et);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Ternary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Ternary.java b/src/main/java/com/ibm/bi/dml/lops/Ternary.java
deleted file mode 100644
index 2feeaac..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Ternary.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform tertiary operation. All inputs must be matrices or vectors.
- * For example, this lop is used in evaluating A = ctable(B,C,W)
- *
- * Currently, this lop is used only in case of CTABLE functionality.
- */
-
-public class Ternary extends Lop
-{
-
- private boolean _ignoreZeros = false;
-
- public enum OperationTypes {
- CTABLE_TRANSFORM,
- CTABLE_TRANSFORM_SCALAR_WEIGHT,
- CTABLE_TRANSFORM_HISTOGRAM,
- CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM,
- CTABLE_EXPAND_SCALAR_WEIGHT,
- INVALID };
-
- OperationTypes operation;
-
-
- public Ternary(Lop[] inputLops, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- this(inputLops, op, dt, vt, false, et);
- }
-
- public Ternary(Lop[] inputLops, OperationTypes op, DataType dt, ValueType vt, boolean ignoreZeros, ExecType et) {
- super(Lop.Type.Ternary, dt, vt);
- init(inputLops, op, et);
- _ignoreZeros = ignoreZeros;
- }
-
- private void init(Lop[] inputLops, OperationTypes op, ExecType et) {
- operation = op;
-
- for(int i=0; i < inputLops.length; i++) {
- this.addInput(inputLops[i]);
- inputLops[i].addOutput(this);
- }
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- //lps.addCompatibility(JobType.DATAGEN); MB: disabled due to piggybacking issues
- //lps.addCompatibility(JobType.REBLOCK); MB: disabled since no runtime support
-
- if( operation==OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT )
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- //TODO create runtime for ctable in gmr mapper and switch to maporreduce.
- //this.lps.setProperties( inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob );
- else
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
-
- return " Operation: " + operation;
-
- }
-
- public static OperationTypes findCtableOperationByInputDataTypes(DataType dt1, DataType dt2, DataType dt3)
- {
- if ( dt1 == DataType.MATRIX ) {
- if (dt2 == DataType.MATRIX && dt3 == DataType.SCALAR) {
- // F = ctable(A,B) or F = ctable(A,B,1)
- return OperationTypes.CTABLE_TRANSFORM_SCALAR_WEIGHT;
- } else if (dt2 == DataType.SCALAR && dt3 == DataType.SCALAR) {
- // F=ctable(A,1) or F = ctable(A,1,1)
- return OperationTypes.CTABLE_TRANSFORM_HISTOGRAM;
- } else if (dt2 == DataType.SCALAR && dt3 == DataType.MATRIX) {
- // F=ctable(A,1,W)
- return OperationTypes.CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM;
- } else {
- // F=ctable(A,B,W)
- return OperationTypes.CTABLE_TRANSFORM;
- }
- }
- else {
- return OperationTypes.INVALID;
- }
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- if( operation != Ternary.OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT )
- sb.append( "ctable" );
- else
- sb.append( "ctableexpand" );
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(0).getDataType() == DataType.SCALAR ) {
- sb.append ( getInputs().get(0).prepScalarInputOperand(getExecType()) );
- }
- else {
- sb.append( getInputs().get(0).prepInputOperand(input1));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(1).getDataType() == DataType.SCALAR ) {
- sb.append ( getInputs().get(1).prepScalarInputOperand(getExecType()) );
- }
- else {
- sb.append( getInputs().get(1).prepInputOperand(input2));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(2).getDataType() == DataType.SCALAR ) {
- sb.append ( getInputs().get(2).prepScalarInputOperand(getExecType()) );
- }
- else {
- sb.append( getInputs().get(2).prepInputOperand(input3));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( this.getInputs().size() > 3 ) {
- sb.append(getInputs().get(3).getOutputParameters().getLabel());
- sb.append(LITERAL_PREFIX);
- sb.append((getInputs().get(3).getType() == Type.Data && ((Data)getInputs().get(3)).isLiteral()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(getInputs().get(4).getOutputParameters().getLabel());
- sb.append(LITERAL_PREFIX);
- sb.append((getInputs().get(4).getType() == Type.Data && ((Data)getInputs().get(4)).isLiteral()) );
- sb.append( OPERAND_DELIMITOR );
- }
- else {
- sb.append(-1);
- sb.append(LITERAL_PREFIX);
- sb.append(true);
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(-1);
- sb.append(LITERAL_PREFIX);
- sb.append(true);
- sb.append( OPERAND_DELIMITOR );
- }
- sb.append( this.prepOutputOperand(output));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _ignoreZeros );
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- switch(operation) {
- /* Arithmetic */
- case CTABLE_TRANSFORM:
- // F = ctable(A,B,W)
- sb.append( "ctabletransform" );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_TRANSFORM_SCALAR_WEIGHT:
- // F = ctable(A,B) or F = ctable(A,B,1)
- // third input must be a scalar, and hence input_index3 == -1
- if ( input_index3 != -1 ) {
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
- }
-
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
- int scalarIndex = 2; // index of the scalar input
-
- sb.append( "ctabletransformscalarweight" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(scalarIndex).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_EXPAND_SCALAR_WEIGHT:
- // F = ctable(seq,B) or F = ctable(seq,B,1)
- // second and third inputs must be scalars, and hence input_index2 == -1, input_index3 == -1
- if ( input_index3 != -1 ) {
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
- }
-
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
- int scalarIndex2 = 1; // index of the scalar input
- int scalarIndex3 = 2; // index of the scalar input
-
- sb.append( "ctableexpandscalarweight" );
- sb.append( OPERAND_DELIMITOR );
- //get(0) because input under group
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(scalarIndex2).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(scalarIndex3).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_TRANSFORM_HISTOGRAM:
- // F=ctable(A,1) or F = ctable(A,1,1)
- if ( input_index2 != -1 || input_index3 != -1)
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation);
-
- // 2nd and 3rd inputs are scalar inputs
-
- sb.append( "ctabletransformhistogram" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- case CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM:
- // F=ctable(A,1,W)
- if ( input_index2 != -1 )
- throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation);
-
- // 2nd input is the scalar input
-
- sb.append( "ctabletransformweightedhistogram" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
- sb.append( OPERAND_DELIMITOR );
-
- break;
-
- default:
- throw new UnsupportedOperationException(this.printErrorLocation() + "Instruction is not defined for Tertiary operation: " + operation);
- }
-
- long outputDim1=-1, outputDim2=-1;
- if ( getInputs().size() > 3 ) {
- sb.append(getInputs().get(3).prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(getInputs().get(4).prepScalarLabel());
- sb.append( OPERAND_DELIMITOR );
- /*if ( input3 instanceof Data && ((Data)input3).isLiteral()
- && input4 instanceof Data && ((Data)input4).isLiteral() ) {
- outputDim1 = ((Data)input3).getLongValue();
- outputDim2 = ((Data)input4).getLongValue();
- }*/
- }
- else {
- sb.append( outputDim1 );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( outputDim2 );
- sb.append( OPERAND_DELIMITOR );
- }
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- /**
- *
- * @param type
- * @return
- */
- public static String getOpcode(OperationTypes type)
- {
- switch( type )
- {
- case CTABLE_TRANSFORM: return "ctabletransform";
- case CTABLE_TRANSFORM_SCALAR_WEIGHT: return "ctabletransformscalarweight";
- case CTABLE_EXPAND_SCALAR_WEIGHT: return "ctableexpandscalarweight";
- case CTABLE_TRANSFORM_HISTOGRAM: return "ctabletransformhistogram";
- case CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM: return "ctabletransformweightedhistogram";
-
- default:
- throw new UnsupportedOperationException("Ternary operation code is not defined: " + type);
- }
- }
-
- /**
- *
- * @param opcode
- * @return
- */
- public static OperationTypes getOperationType(String opcode)
- {
- OperationTypes op = null;
-
- if( opcode.equals("ctabletransform") )
- op = OperationTypes.CTABLE_TRANSFORM;
- else if( opcode.equals("ctabletransformscalarweight") )
- op = OperationTypes.CTABLE_TRANSFORM_SCALAR_WEIGHT;
- else if( opcode.equals("ctableexpandscalarweight") )
- op = OperationTypes.CTABLE_EXPAND_SCALAR_WEIGHT;
- else if( opcode.equals("ctabletransformhistogram") )
- op = OperationTypes.CTABLE_TRANSFORM_HISTOGRAM;
- else if( opcode.equals("ctabletransformweightedhistogram") )
- op = OperationTypes.CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM;
- else
- throw new UnsupportedOperationException("Tertiary operation code is not defined: " + opcode);
-
- return op;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java b/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java
deleted file mode 100644
index 2687b98..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/TernaryAggregate.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- *
- *
- */
-public class TernaryAggregate extends Lop
-{
-
- private static final String OPCODE = "tak+*";
-
- //NOTE: currently only used for ta+*
- //private Aggregate.OperationTypes _aggOp = null;
- //private Binary.OperationTypes _binOp = null;
-
- //optional attribute for cp
- private int _numThreads = -1;
-
- public TernaryAggregate(Lop input1, Lop input2, Lop input3, Aggregate.OperationTypes aggOp, Binary.OperationTypes binOp, DataType dt, ValueType vt, ExecType et ) {
- this(input1, input2, input3, aggOp, binOp, dt, vt, et, 1);
- }
-
- /**
- * @param et
- * @param input - input lop
- * @param op - operation type
- */
- public TernaryAggregate(Lop input1, Lop input2, Lop input3, Aggregate.OperationTypes aggOp, Binary.OperationTypes binOp, DataType dt, ValueType vt, ExecType et, int k )
- {
- super(Lop.Type.TernaryAggregate, dt, vt);
-
- //_aggOp = aggOp;
- //_binOp = binOp;
-
- addInput(input1);
- addInput(input2);
- addInput(input3);
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- _numThreads = k;
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString()
- {
- return "Operation: "+OPCODE;
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepInputOperand(input3));
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output));
-
- if( getExecType() == ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( _numThreads );
- }
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Transform.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Transform.java b/src/main/java/com/ibm/bi/dml/lops/Transform.java
deleted file mode 100644
index b4e6d13..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Transform.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/*
- * Lop to perform transpose/vector to diag operations
- * This lop can change the keys and hence break alignment.
- */
-
-public class Transform extends Lop
-{
-
-
- public enum OperationTypes {
- Transpose,
- Diag,
- Reshape,
- Sort
- };
-
- private boolean _bSortIndInMem = false;
-
- private OperationTypes operation = null;
-
- /**
- * Constructor when we have one input.
- * @param input
- * @param op
- */
-
- public Transform(Lop input, Transform.OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- super(Lop.Type.Transform, dt, vt);
- init(input, op, dt, vt, et);
- }
-
- public Transform(Lop input, Transform.OperationTypes op, DataType dt, ValueType vt)
- {
- super(Lop.Type.Transform, dt, vt);
- init(input, op, dt, vt, ExecType.MR);
- }
-
- public Transform(Lop input, Transform.OperationTypes op, DataType dt, ValueType vt, ExecType et, boolean bSortIndInMem)
- {
- super(Lop.Type.Transform, dt, vt);
- _bSortIndInMem = bSortIndInMem;
- init(input, op, dt, vt, et);
- }
-
- private void init (Lop input, Transform.OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- operation = op;
-
- this.addInput(input);
- input.addOutput(this);
-
- boolean breaksAlignment = true;
- boolean aligner = false;
- boolean definesMRJob = false;
- if ( et == ExecType.MR ) {
- /*
- * This lop CAN NOT be executed in PARTITION, SORT, STANDALONE
- * MMCJ: only in mapper.
- */
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.CSV_REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
-
- if( op == OperationTypes.Reshape )
- //reshape should be executed in map because we have potentially large intermediate data and want to exploit the combiner.
- this.lps.setProperties( inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- else
- this.lps.setProperties( inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob );
- }
- else //CP/SPARK
- {
- // <code>breaksAlignment</code> is not meaningful when <code>Transform</code> executes in CP.
- breaksAlignment = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
-
- return " Operation: " + operation;
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
- private String getOpcode() {
- switch(operation) {
- case Transpose:
- // Transpose a matrix
- return "r'";
-
- case Diag:
- // Transform a vector into a diagonal matrix
- return "rdiag";
-
- case Reshape:
- // Transform a vector into a diagonal matrix
- return "rshape";
-
- case Sort:
- // Transform a matrix into a sorted matrix
- return "rsort";
-
- default:
- throw new UnsupportedOperationException(this.printErrorLocation() + "Instruction is not defined for Transform operation " + operation);
-
- }
- }
-
- //CP instructions
-
- @Override
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String input3, String input4, String output)
- throws LopsException
- {
- //only used for reshape
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- //rows, cols, byrow
- String[] inputX = new String[]{input2,input3,input4};
- for( int i=1; i<=(inputX.length); i++ ) {
- Lop ltmp = getInputs().get(i);
- sb.append( OPERAND_DELIMITOR );
- sb.append( ltmp.prepScalarInputOperand(getExecType()));
- }
-
- //output
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- if( getExecType()==ExecType.SPARK && operation == OperationTypes.Sort ){
- sb.append( OPERAND_DELIMITOR );
- sb.append( _bSortIndInMem);
- }
-
- return sb.toString();
- }
-
- //MR instructions
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index)
- throws LopsException
- {
- //only used for reshape
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
-
- //rows
- Lop input2 = getInputs().get(1);
- String rowsString = input2.prepScalarLabel();
- sb.append( OPERAND_DELIMITOR );
- sb.append( rowsString );
-
- //cols
- Lop input3 = getInputs().get(2);
- String colsString = input3.prepScalarLabel();
- sb.append( OPERAND_DELIMITOR );
- sb.append( colsString );
-
- //byrow
- Lop input4 = getInputs().get(3);
- String byrowString = input4.prepScalarLabel();
- if ( input4.getExecLocation() == ExecLocation.Data
- && !((Data)input4).isLiteral() || !(input4.getExecLocation() == ExecLocation.Data ) ){
- throw new LopsException(this.printErrorLocation() + "Parameter 'byRow' must be a literal for a matrix operation.");
- }
- sb.append( OPERAND_DELIMITOR );
- sb.append( byrowString );
-
- //output
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- public static Transform constructTransformLop(Lop input1, OperationTypes op, DataType dt, ValueType vt) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.Transform ) {
- return (Transform)lop;
- }
- }
- Transform retVal = new Transform(input1, op, dt, vt);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
- public static Transform constructTransformLop(Lop input1, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.Transform ) {
- return (Transform)lop;
- }
- }
- Transform retVal = new Transform(input1, op, dt, vt, et);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java b/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java
deleted file mode 100644
index fd36c9b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/UAggOuterChain.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * TODO Currently this lop only support the right hand side in distributed cache. This
- * should be generalized (incl hop operator selection) to left/right cache types.
- *
- *
- */
-public class UAggOuterChain extends Lop
-{
-
- public static final String OPCODE = "uaggouterchain";
-
- //outer operation
- private Aggregate.OperationTypes _uaggOp = null;
- private PartialAggregate.DirectionTypes _uaggDir = null;
- //inner operation
- private Binary.OperationTypes _binOp = null;
-
-
- /**
- * Constructor to setup a unaryagg outer chain
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public UAggOuterChain(Lop input1, Lop input2, Aggregate.OperationTypes uaop, PartialAggregate.DirectionTypes uadir, Binary.OperationTypes bop, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.UaggOuterChain, dt, vt);
- addInput(input1);
- addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- //setup operator types
- _uaggOp = uaop;
- _uaggDir = uadir;
- _binOp = bop;
-
- //setup MR parameters
- if( et == ExecType.MR )
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.CSV_REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //SPARK
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
-
- public String toString() {
- return "Operation = UaggOuterChain";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( getInputs().get(0).prepInputOperand(input2));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
-
- @Override
- public boolean usesDistributedCache()
- {
- return true;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- return new int[]{2};
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Unary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Unary.java b/src/main/java/com/ibm/bi/dml/lops/Unary.java
deleted file mode 100644
index b29e822..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Unary.java
+++ /dev/null
@@ -1,445 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to perform following operations: with one operand -- NOT(A), ABS(A),
- * SQRT(A), LOG(A) with two operands where one of them is a scalar -- H=H*i,
- * H=H*5, EXP(A,2), LOG(A,2)
- *
- */
-
-public class Unary extends Lop
-{
-
- public enum OperationTypes {
- ADD, SUBTRACT, SUBTRACTRIGHT, MULTIPLY, MULTIPLY2, DIVIDE, MODULUS, INTDIV, MINUS1_MULTIPLY, POW, POW2, LOG, MAX, MIN, NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, EXP, Over, LESS_THAN, LESS_THAN_OR_EQUALS, GREATER_THAN, GREATER_THAN_OR_EQUALS, EQUALS, NOT_EQUALS, ROUND, CEIL, FLOOR, MR_IQM, INVERSE,
- CUMSUM, CUMPROD, CUMMIN, CUMMAX,
- SPROP, SIGMOID, SELP, SUBTRACT_NZ, LOG_NZ,
- NOTSUPPORTED
- };
-
- OperationTypes operation;
-
- Lop valInput;
-
- /**
- * Constructor to perform a unary operation with 2 inputs
- *
- * @param input
- * @param op
- */
-
- public Unary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, input2, op, dt, vt, et);
- }
-
- public Unary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, input2, op, dt, vt, ExecType.MR);
- }
-
- private void init(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- operation = op;
-
- if (input1.getDataType() == DataType.MATRIX)
- valInput = input2;
- else
- valInput = input1;
-
- this.addInput(input1);
- input1.addOutput(this);
- this.addInput(input2);
- input2.addOutput(this);
-
- // By definition, this lop should not break alignment
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- /*
- * This lop CAN NOT be executed in PARTITION, SORT, CM_COV, and COMBINE
- * jobs MMCJ: only in mapper.
- */
- lps.addCompatibility(JobType.ANY);
- lps.removeNonPiggybackableJobs();
- lps.removeCompatibility(JobType.CM_COV); // CM_COV allows only reducer instructions but this is MapOrReduce. TODO: piggybacking should be updated to take this extra constraint.
- lps.removeCompatibility(JobType.TRANSFORM);
- this.lps.setProperties(inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- /**
- * Constructor to perform a unary operation with 1 input.
- *
- * @param input1
- * @param op
- */
- public Unary(Lop input1, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, op, dt, vt, et);
- }
-
- public Unary(Lop input1, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.UNARY, dt, vt);
- init(input1, op, dt, vt, ExecType.MR);
- }
-
- private ExecType forceExecType(OperationTypes op, ExecType et) {
- if ( op == OperationTypes.INVERSE )
- return ExecType.CP;
- return et;
- }
- private void init(Lop input1, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- operation = op;
-
- et = forceExecType(op, et);
-
- valInput = null;
-
- this.addInput(input1);
- input1.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- /*
- * This lop can be executed in all jobs except for PARTITION. MMCJ: only
- * in mapper. GroupedAgg: only in reducer.
- */
- lps.addCompatibility(JobType.ANY);
- lps.removeNonPiggybackableJobs();
- lps.removeCompatibility(JobType.CM_COV); // CM_COV allows only reducer instructions but this is MapOrReduce. TODO: piggybacking should be updated to take this extra constraint.
- lps.removeCompatibility(JobType.TRANSFORM);
- this.lps.setProperties(inputs, et, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob);
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- @Override
- public String toString() {
- if (valInput != null)
- return "Operation: " + operation + " " + "Label: "
- + valInput.getOutputParameters().getLabel()
- + " input types " + this.getInputs().get(0).toString()
- + " " + this.getInputs().get(1).toString();
- else
- return "Operation: " + operation + " " + "Label: N/A";
- }
-
- /**
- *
- * @return
- * @throws LopsException
- */
- private String getOpcode()
- throws LopsException
- {
- return getOpcode(operation);
- }
-
- /**
- *
- * @param op
- * @return
- * @throws LopsException
- */
- public static String getOpcode(OperationTypes op)
- throws LopsException
- {
- switch (op) {
- case NOT:
- return "!";
- case ABS:
- return "abs";
- case SIN:
- return "sin";
- case COS:
- return "cos";
- case TAN:
- return "tan";
- case ASIN:
- return "asin";
- case ACOS:
- return "acos";
- case ATAN:
- return "atan";
- case SQRT:
- return "sqrt";
- case EXP:
- return "exp";
-
- case LOG:
- return "log";
-
- case LOG_NZ:
- return "log_nz";
-
- case ROUND:
- return "round";
-
- case ADD:
- return "+";
-
- case SUBTRACT:
- return "-";
-
- case SUBTRACT_NZ:
- return "-nz";
-
- case SUBTRACTRIGHT:
- return "s-r";
-
- case MULTIPLY:
- return "*";
-
- case MULTIPLY2:
- return "*2";
-
- case MINUS1_MULTIPLY:
- return "1-*";
-
- case DIVIDE:
- return "/";
-
- case MODULUS:
- return "%%";
-
- case INTDIV:
- return "%/%";
-
- case Over:
- return "so";
-
- case POW:
- return "^";
-
- case POW2:
- return "^2";
-
- case GREATER_THAN:
- return ">";
-
- case GREATER_THAN_OR_EQUALS:
- return ">=";
-
- case LESS_THAN:
- return "<";
-
- case LESS_THAN_OR_EQUALS:
- return "<=";
-
- case EQUALS:
- return "==";
-
- case NOT_EQUALS:
- return "!=";
-
- case MAX:
- return "max";
-
- case MIN:
- return "min";
-
- case CEIL:
- return "ceil";
-
- case FLOOR:
- return "floor";
-
- case CUMSUM:
- return "ucumk+";
-
- case CUMPROD:
- return "ucum*";
-
- case CUMMIN:
- return "ucummin";
-
- case CUMMAX:
- return "ucummax";
-
- case INVERSE:
- return "inverse";
-
- case MR_IQM:
- return "qpick";
-
- case SPROP:
- return "sprop";
-
- case SIGMOID:
- return "sigmoid";
-
- case SELP:
- return "sel+";
-
- default:
- throw new LopsException(
- "Instruction not defined for Unary operation: " + op);
- }
- }
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- // Unary operators with one input
- if (this.getInputs().size() == 1) {
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for an Unary opration: "
- + operation);
- }
- }
-
- @Override
- public String getInstructions(int input_index, int output_index)
- throws LopsException {
- return getInstructions(""+input_index, ""+output_index);
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(0).getDataType() == DataType.SCALAR ) {
- sb.append( getInputs().get(0).prepScalarInputOperand(getExecType()));
- }
- else {
- sb.append( getInputs().get(0).prepInputOperand(input1));
- }
- sb.append( OPERAND_DELIMITOR );
-
- if ( getInputs().get(1).getDataType() == DataType.SCALAR ) {
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- }
- else {
- sb.append( getInputs().get(1).prepInputOperand(input2));
- }
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int inputIndex1, int inputIndex2,
- int outputIndex) throws LopsException {
- if (this.getInputs().size() == 2) {
- // Unary operators with two inputs
- // Determine the correct operation, depending on the scalar input
- Lop linput1 = getInputs().get(0);
- Lop linput2 = getInputs().get(1);
-
- int scalarIndex = -1, matrixIndex = -1;
- String matrixLabel= null;
- if( linput1.getDataType() == DataType.MATRIX ) {
- // inputIndex1 is matrix, and inputIndex2 is scalar
- scalarIndex = 1;
- matrixLabel = String.valueOf(inputIndex1);
- }
- else {
- // inputIndex2 is matrix, and inputIndex1 is scalar
- scalarIndex = 0;
- matrixLabel = String.valueOf(inputIndex2);
-
- // when the first operand is a scalar, setup the operation type accordingly
- if (operation == OperationTypes.SUBTRACT)
- operation = OperationTypes.SUBTRACTRIGHT;
- else if (operation == OperationTypes.DIVIDE)
- operation = OperationTypes.Over;
- }
- matrixIndex = 1-scalarIndex;
-
- // Prepare the instruction
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- if( operation == OperationTypes.INTDIV || operation == OperationTypes.MODULUS ||
- operation == OperationTypes.POW ||
- operation == OperationTypes.GREATER_THAN || operation == OperationTypes.GREATER_THAN_OR_EQUALS ||
- operation == OperationTypes.LESS_THAN || operation == OperationTypes.LESS_THAN_OR_EQUALS ||
- operation == OperationTypes.EQUALS || operation == OperationTypes.NOT_EQUALS )
- {
- //TODO discuss w/ Shirish: we should consolidate the other operations (see ScalarInstruction.parseInstruction / BinaryCPInstruction.getScalarOperator)
- //append both operands
- sb.append( (linput1.getDataType()==DataType.MATRIX? linput1.prepInputOperand(String.valueOf(inputIndex1)) : linput1.prepScalarInputOperand(getExecType())) );
- sb.append( OPERAND_DELIMITOR );
- sb.append( (linput2.getDataType()==DataType.MATRIX? linput2.prepInputOperand(String.valueOf(inputIndex2)) : linput2.prepScalarInputOperand(getExecType())) );
- sb.append( OPERAND_DELIMITOR );
- }
- else
- {
- // append the matrix operand
- sb.append( getInputs().get(matrixIndex).prepInputOperand(matrixLabel));
- sb.append( OPERAND_DELIMITOR );
-
- // append the scalar operand
- sb.append( getInputs().get(scalarIndex).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
- }
- sb.append( this.prepOutputOperand(outputIndex+""));
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for an Unary opration: "
- + operation);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java b/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java
deleted file mode 100644
index c92fbfc..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/UnaryCP.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform unary scalar operations. Example a = !b
- *
- */
-
-public class UnaryCP extends Lop
-{
-
- public enum OperationTypes {
- NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, LOG, EXP, CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, PRINT, NROW, NCOL, LENGTH, ROUND, STOP, CEIL, FLOOR, CUMSUM, NOTSUPPORTED
- };
-
- public static final String CAST_AS_SCALAR_OPCODE = "castdts";
- public static final String CAST_AS_MATRIX_OPCODE = "castdtm";
- public static final String CAST_AS_DOUBLE_OPCODE = "castvtd";
- public static final String CAST_AS_INT_OPCODE = "castvti";
- public static final String CAST_AS_BOOLEAN_OPCODE = "castvtb";
-
-
-
- OperationTypes operation;
-
- /**
- * Constructor to perform a scalar operation
- *
- * @param input
- * @param op
- */
-
- public UnaryCP(Lop input, OperationTypes op, DataType dt, ValueType vt) {
- super(Lop.Type.UnaryCP, dt, vt);
- operation = op;
- this.addInput(input);
- input.addOutput(this);
-
- //This lop is executed in control program.
- boolean breaksAlignment = false; // this does not carry any information
- // for this lop
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
-
- @Override
- public String toString() {
-
- return "Operation: " + operation;
-
- }
-
- private String getOpCode() throws LopsException {
- switch (operation) {
- case NOT:
- return "!";
-
- case ABS:
- return "abs";
-
- case SIN:
- return "sin";
-
- case COS:
- return "cos";
-
- case TAN:
- return "tan";
-
- case ASIN:
- return "asin";
-
- case ACOS:
- return "acos";
-
- case ATAN:
- return "atan";
-
- case SQRT:
- return "sqrt";
-
- case LOG:
- return "log";
-
- case ROUND:
- return "round";
-
- case EXP:
- return "exp";
-
- case PRINT:
- return "print";
-
- case CAST_AS_MATRIX:
- return CAST_AS_MATRIX_OPCODE;
-
- case STOP:
- return "stop";
-
- case CEIL:
- return "ceil";
-
- case FLOOR:
- return "floor";
-
- case CUMSUM:
- return "ucumk+";
-
- // CAST_AS_SCALAR, NROW, NCOL, LENGTH builtins take matrix as the input
- // and produces a scalar
- case CAST_AS_SCALAR:
- return CAST_AS_SCALAR_OPCODE;
-
- case CAST_AS_DOUBLE:
- return CAST_AS_DOUBLE_OPCODE;
-
- case CAST_AS_INT:
- return CAST_AS_INT_OPCODE;
-
- case CAST_AS_BOOLEAN:
- return CAST_AS_BOOLEAN_OPCODE;
-
- case NROW:
- return "nrow";
-
- case NCOL:
- return "ncol";
-
- case LENGTH:
- return "length";
-
- default:
- throw new LopsException(this.printErrorLocation() + "Unknown operation: " + operation);
- }
- }
-
- @Override
- public String getInstructions(String input, String output)
- throws LopsException {
- StringBuilder sb = new StringBuilder();
- sb.append(getExecType());
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpCode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append(getInputs().get(0).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
-
- }
-}