You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:46:00 UTC
[29/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
deleted file mode 100644
index 799f8ab..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.hops.Hop.VisitStatus;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.StatementBlock;
-import com.ibm.bi.dml.parser.VariableSet;
-
-/**
- * Rule: Split Hop DAG after CSV reads with unknown size. This is
- * important to create recompile hooks if format is read from mtd
- * (we are not able to split it on statementblock creation) and
- * mtd has unknown size (which can only happen for CSV).
- *
- */
-public class RewriteSplitDagUnknownCSVRead extends StatementBlockRewriteRule
-{
-
- @Override
- public ArrayList<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state)
- throws HopsException
- {
- ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
-
- //collect all unknown csv reads hops
- ArrayList<Hop> cand = new ArrayList<Hop>();
- collectCSVReadHopsUnknownSize( sb.get_hops(), cand );
-
- //split hop dag on demand
- if( !cand.isEmpty() )
- {
- try
- {
- //duplicate sb incl live variable sets
- StatementBlock sb1 = new StatementBlock();
- sb1.setDMLProg(sb.getDMLProg());
- sb1.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- sb1.setLiveIn(new VariableSet());
- sb1.setLiveOut(new VariableSet());
-
- //move csv reads incl reblock to new statement block
- //(and replace original persistent read with transient read)
- ArrayList<Hop> sb1hops = new ArrayList<Hop>();
- for( Hop c : cand )
- {
- Hop reblock = c;
- long rlen = reblock.getDim1();
- long clen = reblock.getDim2();
- long nnz = reblock.getNnz();
- long brlen = reblock.getRowsInBlock();
- long bclen = reblock.getColsInBlock();
-
- //create new transient read
- DataOp tread = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(),
- DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, brlen, bclen);
- HopRewriteUtils.copyLineNumbers(reblock, tread);
-
- //replace reblock with transient read
- ArrayList<Hop> parents = new ArrayList<Hop>(reblock.getParent());
- for( int i=0; i<parents.size(); i++ )
- {
- Hop parent = parents.get(i);
- int pos = HopRewriteUtils.getChildReferencePos(parent, reblock);
- HopRewriteUtils.removeChildReferenceByPos(parent, reblock, pos);
- HopRewriteUtils.addChildReference(parent, tread, pos);
- }
-
- //add reblock sub dag to first statement block
- DataOp twrite = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(),
- reblock, DataOpTypes.TRANSIENTWRITE, null);
- twrite.setOutputParams(rlen, clen, nnz, brlen, bclen);
- HopRewriteUtils.copyLineNumbers(reblock, twrite);
- sb1hops.add(twrite);
-
- //update live in and out of new statement block (for piggybacking)
- DataIdentifier diVar = sb.variablesRead().getVariable(reblock.getName());
- if( diVar != null ){ //var read should always exist because persistent read
- sb1.liveOut().addVariable(reblock.getName(), new DataIdentifier(diVar));
- sb.liveIn().addVariable(reblock.getName(), new DataIdentifier(diVar));
- }
- }
-
- sb1.set_hops(sb1hops);
- sb1.updateRecompilationFlag();
- ret.add(sb1); //statement block with csv reblocks
- ret.add(sb); //statement block with remaining hops
- }
- catch(Exception ex)
- {
- throw new HopsException("Failed to split hops dag for csv read with unknown size.", ex);
- }
- LOG.debug("Applied splitDagUnknownCSVRead.");
- }
- //keep original hop dag
- else
- {
- ret.add(sb);
- }
-
- return ret;
- }
-
- /**
- *
- * @param roots
- * @param cand
- */
- private void collectCSVReadHopsUnknownSize( ArrayList<Hop> roots, ArrayList<Hop> cand )
- {
- if( roots == null )
- return;
-
- Hop.resetVisitStatus(roots);
- for( Hop root : roots )
- collectCSVReadHopsUnknownSize(root, cand);
- }
-
- /**
- *
- * @param root
- * @param cand
- */
- private void collectCSVReadHopsUnknownSize( Hop hop, ArrayList<Hop> cand )
- {
- if( hop.getVisited() == VisitStatus.DONE )
- return;
-
- //collect persistent reads (of type csv, with unknown size)
- if( hop instanceof DataOp )
- {
- DataOp dop = (DataOp) hop;
- if( dop.getDataOpType() == DataOpTypes.PERSISTENTREAD
- && dop.getInputFormatType() == FileFormatTypes.CSV
- && !dop.dimsKnown()
- && !HopRewriteUtils.hasOnlyWriteParents(dop, true, false)
- && !HopRewriteUtils.hasTransformParents(hop) )
- {
- cand.add(dop);
- }
- }
-
- //process children
- if( hop.getInput()!=null )
- for( Hop c : hop.getInput() )
- collectCSVReadHopsUnknownSize(c, cand);
-
- hop.setVisited(VisitStatus.DONE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java
deleted file mode 100644
index ee9c010..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteTransientWriteParentHandling.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-
-/**
- * Rule: Eliminate for Transient Write DataHops to have no parents
- * Solution: Move parent edges of Transient Write Hop to parent of
- * its child
- * Reason: Transient Write not being a root messes up
- * analysis for Lop's to Instruction translation (according to Amol)
- */
-public class RewriteTransientWriteParentHandling extends HopRewriteRule
-{
-
- @Override
- public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
- throws HopsException
- {
- for (Hop h : roots)
- rule_RehangTransientWriteParents(h, roots);
-
- return roots;
- }
-
- @Override
- public Hop rewriteHopDAG(Hop root, ProgramRewriteStatus state)
- throws HopsException
- {
- // do nothing (does not apply to predicate hop DAGs)
- return root;
- }
-
-
- /**
- *
- * @param hop
- * @param sbHops
- * @throws HopsException
- */
- private void rule_RehangTransientWriteParents(Hop hop, ArrayList<Hop> sbHops)
- throws HopsException
- {
- if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTWRITE
- && !hop.getParent().isEmpty()) {
-
- // update parents inputs with data op input
- for (Hop p : hop.getParent()) {
- p.getInput().set(p.getInput().indexOf(hop), hop.getInput().get(0));
- }
-
- // update dataop input parent to add new parents except for
- // dataop itself
- hop.getInput().get(0).getParent().addAll(hop.getParent());
-
- // remove dataop parents
- hop.getParent().clear();
-
- // add dataop as root for this Hops DAG
- sbHops.add(hop);
-
- // do the same thing for my inputs (children)
- for (Hop hi : hop.getInput()) {
- rule_RehangTransientWriteParents(hi,sbHops);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java
deleted file mode 100644
index 0e4c7b8..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/StatementBlockRewriteRule.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops.rewrite;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.parser.StatementBlock;
-
-/**
- * Base class for all hop rewrites in order to enable generic
- * application of all rules.
- *
- */
-public abstract class StatementBlockRewriteRule
-{
-
- protected static final Log LOG = LogFactory.getLog(StatementBlockRewriteRule.class.getName());
-
- /**
- * Handle an arbitrary statement block. Specific type constraints have to be ensured
- * within the individual rewrites.
- *
- * @param sb
- * @param sate
- * @return
- * @throws HopsException
- */
- public abstract ArrayList<StatementBlock> rewriteStatementBlock( StatementBlock sb, ProgramRewriteStatus sate )
- throws HopsException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Aggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Aggregate.java b/src/main/java/com/ibm/bi/dml/lops/Aggregate.java
deleted file mode 100644
index 88d57e7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Aggregate.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.PartialAggregate.CorrectionLocationType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to represent an aggregation.
- * It is used in rowsum, colsum, etc.
- */
-
-public class Aggregate extends Lop
-{
-
-
- /** Aggregate operation types **/
-
- public enum OperationTypes {
- Sum, Product, Min, Max, Trace, KahanSum, KahanSumSq, KahanTrace, Mean,MaxIndex, MinIndex
- }
- OperationTypes operation;
-
- private boolean isCorrectionUsed = false;
- private CorrectionLocationType correctionLocation = CorrectionLocationType.INVALID;
-
- /**
- * @param input - input lop
- * @param op - operation type
- */
- public Aggregate(Lop input, Aggregate.OperationTypes op, DataType dt, ValueType vt ) {
- super(Lop.Type.Aggregate, dt, vt);
- init ( input, op, dt, vt, ExecType.MR );
- }
-
- public Aggregate(Lop input, Aggregate.OperationTypes op, DataType dt, ValueType vt, ExecType et ) {
- super(Lop.Type.Aggregate, dt, vt);
- init ( input, op, dt, vt, et );
- }
-
- private void init (Lop input, Aggregate.OperationTypes op, DataType dt, ValueType vt, ExecType et ) {
- operation = op;
- this.addInput(input);
- input.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else {
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- // this function must be invoked during hop-to-lop translation
- public void setupCorrectionLocation(CorrectionLocationType loc) {
- if (operation == OperationTypes.KahanSum || operation == OperationTypes.KahanSumSq
- || operation == OperationTypes.KahanTrace || operation == OperationTypes.Mean) {
- isCorrectionUsed = true;
- correctionLocation = loc;
- }
- }
-
- /**
- * for debugging purposes.
- */
-
- public String toString()
- {
- return "Operation: " + operation;
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
-
- private String getOpcode() {
- switch(operation) {
- case Sum:
- case Trace:
- return "a+";
- case Mean:
- return "amean";
- case Product:
- return "a*";
- case Min:
- return "amin";
- case Max:
- return "amax";
- case MaxIndex:
- return "arimax";
- case MinIndex:
- return "arimin";
- case KahanSum:
- case KahanTrace:
- return "ak+";
- case KahanSumSq:
- return "asqk+";
- default:
- throw new UnsupportedOperationException(this.printErrorLocation() + "Instruction is not defined for Aggregate operation: " + operation);
- }
- }
-
- @Override
- public String getInstructions(String input1, String output) throws LopsException {
- String opcode = getOpcode();
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( opcode );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index, int output_index) throws LopsException
- {
- boolean isCorrectionApplicable = false;
-
- String opcode = getOpcode();
- if (operation == OperationTypes.Mean || operation == OperationTypes.KahanSum
- || operation == OperationTypes.KahanSumSq || operation == OperationTypes.KahanTrace)
- isCorrectionApplicable = true;
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( opcode );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index));
-
- if ( isCorrectionApplicable )
- {
- // add correction information to the instruction
- sb.append( OPERAND_DELIMITOR );
- sb.append( isCorrectionUsed );
- sb.append( OPERAND_DELIMITOR );
- sb.append( correctionLocation );
- }
-
- return sb.toString();
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendCP.java b/src/main/java/com/ibm/bi/dml/lops/AppendCP.java
deleted file mode 100644
index f3108e5..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendCP.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-public class AppendCP extends Lop
-{
- public static final String OPCODE = "append";
-
- private boolean _cbind = true;
-
- public AppendCP(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, boolean cbind)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, dt, vt);
-
- _cbind = cbind;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
-
- return " AppendCP: ";
- }
-
- //called when append executes in CP
- public String getInstructions(String input_index1, String input_index2, String input_index3, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( "append" );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendG.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendG.java b/src/main/java/com/ibm/bi/dml/lops/AppendG.java
deleted file mode 100644
index f254aa0..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendG.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-/**
- * TODO Additional compiler enhancements:
- * 1) Partial Shuffle Elimination - Any full or aligned blocks could be directly output from the mappers
- * to the result index. We only need to shuffle, sort and aggregate partial blocks. However, this requires
- * piggybacking changes, i.e., (1) operations with multiple result indexes, and (2) multiple operations
- * with the same result index.
- * 2) Group Elimination for Append Chains - If we have chains of rappend each intermediate is shuffled and
- * aggregated. This is unnecessary if all offsets are known in advance. We could directly pack all rappends
- * in one GMR map-phase followed by one group and subsequent aggregate. However, this requires an n-ary
- * rappend or group (with multiple inputs).
- *
- */
-public class AppendG extends Lop
-{
- public static final String OPCODE = "gappend";
-
- private boolean _cbind = true;
-
- public AppendG(Lop input1, Lop input2, Lop input3, Lop input4, DataType dt, ValueType vt, boolean cbind, ExecType et)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, input4, dt, vt, et);
-
- _cbind = cbind;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, Lop input4, DataType dt, ValueType vt, ExecType et)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- addInput(input4);
- input4.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //SP
- {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return " AppendG: ";
- }
-
- //called when append executes in MR
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input_index1),
- String.valueOf(input_index2),
- String.valueOf(input_index3),
- String.valueOf(input_index4),
- String.valueOf(output_index) );
- }
-
- //called when append executes in SP
- public String getInstructions(String input_index1, String input_index2, String input_index3, String input_index4, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(3).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java b/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java
deleted file mode 100644
index b1b86c1..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendGAlignedSP.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-public class AppendGAlignedSP extends Lop
-{
- public static final String OPCODE = "galignedappend";
-
- private boolean _cbind = true;
-
- public AppendGAlignedSP(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, boolean cbind)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, dt, vt);
-
- _cbind = true;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
- return " AppendGSP: ";
- }
-
- //called when append executes in CP
- public String getInstructions(String input_index1, String input_index2, String input_index3, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendM.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendM.java b/src/main/java/com/ibm/bi/dml/lops/AppendM.java
deleted file mode 100644
index ab364ca..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendM.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-public class AppendM extends Lop
-{
- public static final String OPCODE = "mappend";
-
- public enum CacheType {
- RIGHT,
- RIGHT_PART,
- }
-
- private boolean _cbind = true;
- private CacheType _cacheType = null;
-
- public AppendM(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, boolean cbind, boolean partitioned, ExecType et)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, input3, dt, vt, et);
-
- _cbind = cbind;
- _cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
- }
-
- public void init(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- addInput(input3);
- input3.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else //SPARK
- {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return "Operation = AppendM";
- }
-
- //called when append executes in MR
- public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input_index1),
- String.valueOf(input_index2),
- String.valueOf(input_index3),
- String.valueOf(output_index) );
- }
-
-
- //called when append executes in SP
- public String getInstructions(String input1, String input2, String input3, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input2));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output) );
-
- //note: for SP: no cache type
- if( getExecType()==ExecType.MR ){
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(_cacheType);
- }
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-
- public boolean usesDistributedCache() {
- return true;
- }
-
- public int[] distributedCacheInputIndex() {
- return new int[]{2}; // second input is from distributed cache
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/AppendR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/AppendR.java b/src/main/java/com/ibm/bi/dml/lops/AppendR.java
deleted file mode 100644
index 94b83bf..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/AppendR.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-public class AppendR extends Lop
-{
- public static final String OPCODE = "rappend";
-
- private boolean _cbind = true;
-
- public AppendR(Lop input1, Lop input2, DataType dt, ValueType vt, boolean cbind, ExecType et)
- {
- super(Lop.Type.Append, dt, vt);
- init(input1, input2, dt, vt, et);
-
- _cbind = cbind;
- }
-
- public void init(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et)
- {
- addInput(input1);
- input1.addOutput(this);
-
- addInput(input2);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN); //currently required for correctness
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else //SP
- {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
- return " AppendR: ";
- }
-
- //called when append executes in MR
- public String getInstructions(int input_index1, int input_index2, int output_index)
- throws LopsException
- {
- return getInstructions(
- String.valueOf(input_index1),
- String.valueOf(input_index2),
- String.valueOf(output_index) );
- }
-
- //called when append executes in CP
- public String getInstructions(String input_index1, String input_index2, String output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( OPCODE );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input_index1+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(1).prepInputOperand(input_index2+""));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output_index+"") );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( _cbind );
-
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Binary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Binary.java b/src/main/java/com/ibm/bi/dml/lops/Binary.java
deleted file mode 100644
index 1ceabda..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Binary.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform binary operation. Both inputs must be matrices or vectors.
- * Example - A = B + C, where B and C are matrices or vectors.
- */
-
-public class Binary extends Lop
-{
-
- public enum OperationTypes {
- ADD, SUBTRACT, MULTIPLY, DIVIDE, MINUS1_MULTIPLY, MODULUS, INTDIV, MATMULT,
- LESS_THAN, LESS_THAN_OR_EQUALS, GREATER_THAN, GREATER_THAN_OR_EQUALS, EQUALS, NOT_EQUALS,
- AND, OR,
- MAX, MIN, POW, SOLVE, NOTSUPPORTED
- };
-
- private OperationTypes operation;
- private int numThreads = -1;
-
- /**
- * Constructor to perform a binary operation.
- * @param input
- * @param op
- */
- public Binary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
- this(input1, input2, op, dt, vt, et, 1);
- }
-
- public Binary(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et, int k) {
- super(Lop.Type.Binary, dt, vt);
- init(input1, input2, op, dt, vt, et);
- numThreads = k;
- }
-
- private void init(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if ( et == ExecType.MR ) {
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- this.lps.setProperties( inputs, et, ExecLocation.Reduce, breaksAlignment, aligner, definesMRJob );
- }
- else if ( et == ExecType.CP || et == ExecType.SPARK ){
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- }
-
- @Override
- public String toString() {
-
- return " Operation: " + operation;
-
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return operation;
- }
-
- private String getOpcode()
- {
- return getOpcode( operation );
- }
-
- public static String getOpcode( OperationTypes op ) {
- switch(op) {
- /* Arithmetic */
- case ADD:
- return "+";
- case SUBTRACT:
- return "-";
- case MULTIPLY:
- return "*";
- case DIVIDE:
- return "/";
- case MODULUS:
- return "%%";
- case INTDIV:
- return "%/%";
- case MATMULT:
- return "ba+*";
- case MINUS1_MULTIPLY:
- return "1-*";
-
- /* Relational */
- case LESS_THAN:
- return "<";
- case LESS_THAN_OR_EQUALS:
- return "<=";
- case GREATER_THAN:
- return ">";
- case GREATER_THAN_OR_EQUALS:
- return ">=";
- case EQUALS:
- return "==";
- case NOT_EQUALS:
- return "!=";
-
- /* Boolean */
- case AND:
- return "&&";
- case OR:
- return "||";
-
-
- /* Builtin Functions */
- case MIN:
- return "min";
- case MAX:
- return "max";
- case POW:
- return "^";
-
- case SOLVE:
- return "solve";
-
- default:
- throw new UnsupportedOperationException("Instruction is not defined for Binary operation: " + op);
- }
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append ( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append ( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- //append degree of parallelism for matrix multiplications
- if( operation == OperationTypes.MATMULT && getExecType()==ExecType.CP ) {
- sb.append( OPERAND_DELIMITOR );
- sb.append( numThreads );
- }
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index) throws LopsException
- {
- return getInstructions(input_index1+"", input_index2+"", output_index+"");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/BinaryM.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/BinaryM.java b/src/main/java/com/ibm/bi/dml/lops/BinaryM.java
deleted file mode 100644
index 5b03008..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/BinaryM.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.Binary.OperationTypes;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform binary operation. Both inputs must be matrices or vectors.
- * Example - A = B + C, where B and C are matrices or vectors.
- */
-
-public class BinaryM extends Lop
-{
-
- public enum CacheType {
- RIGHT,
- RIGHT_PART,
- }
-
- public enum VectorType{
- COL_VECTOR,
- ROW_VECTOR,
- }
-
- private OperationTypes _operation;
- private CacheType _cacheType = null;
- private VectorType _vectorType = null;
-
- /**
- * Constructor to perform a binary operation.
- * @param input
- * @param op
- * @throws DMLRuntimeException
- */
-
- public BinaryM(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et, boolean partitioned, boolean colVector ) throws LopsException {
- super(Lop.Type.Binary, dt, vt);
-
- _operation = op;
- _cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
- _vectorType = colVector ? VectorType.COL_VECTOR : VectorType.ROW_VECTOR;
-
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- if(et == ExecType.MR) {
- lps.addCompatibility(JobType.GMR);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
- }
- else if(et == ExecType.SPARK) {
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- else {
- throw new LopsException("Incorrect execution type for BinaryM lop:" + et.name());
- }
- }
-
-
- @Override
- public String toString()
- {
- return " Operation: " + _operation;
- }
-
- /**
- * method to get operation type
- * @return
- */
-
- public OperationTypes getOperationType()
- {
- return _operation;
- }
-
- private String getOpcode()
- {
- return getOpcode( _operation );
- }
-
- public static String getOpcode( OperationTypes op ) {
- switch(op) {
- /* Arithmetic */
- case ADD:
- return "map+";
- case SUBTRACT:
- return "map-";
- case MULTIPLY:
- return "map*";
- case DIVIDE:
- return "map/";
- case MODULUS:
- return "map%%";
- case INTDIV:
- return "map%/%";
- case MINUS1_MULTIPLY:
- return "map1-*";
-
- /* Relational */
- case LESS_THAN:
- return "map<";
- case LESS_THAN_OR_EQUALS:
- return "map<=";
- case GREATER_THAN:
- return "map>";
- case GREATER_THAN_OR_EQUALS:
- return "map>=";
- case EQUALS:
- return "map==";
- case NOT_EQUALS:
- return "map!=";
-
- /* Boolean */
- case AND:
- return "map&&";
- case OR:
- return "map||";
-
-
- /* Builtin Functions */
- case MIN:
- return "mapmin";
- case MAX:
- return "mapmax";
- case POW:
- return "map^";
-
- default:
- throw new UnsupportedOperationException("Instruction is not defined for Binary operation: " + op);
- }
- }
-
- public static boolean isOpcode(String opcode)
- {
- return opcode.equals("map+") || opcode.equals("map-") ||
- opcode.equals("map*") || opcode.equals("map/") ||
- opcode.equals("map%%") || opcode.equals("map%/%") ||
- opcode.equals("map<") || opcode.equals("map<=") ||
- opcode.equals("map>") || opcode.equals("map>=") ||
- opcode.equals("map==") || opcode.equals("map!=") ||
- opcode.equals("map&&") || opcode.equals("map||") ||
- opcode.equals("mapmin") || opcode.equals("mapmax") ||
- opcode.equals("map^") || opcode.equals("map1-*");
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( getOpcode() );
-
- sb.append( OPERAND_DELIMITOR );
- sb.append ( getInputs().get(0).prepInputOperand(input1));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append ( getInputs().get(1).prepInputOperand(input2));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output));
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(_cacheType);
-
- sb.append( OPERAND_DELIMITOR );
- sb.append(_vectorType);
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index) throws LopsException
- {
- return getInstructions(input_index1+"", input_index2+"", output_index+"");
- }
-
- @Override
- public boolean usesDistributedCache()
- {
- return true;
- }
-
- @Override
- public int[] distributedCacheInputIndex()
- {
- // second input is from distributed cache
- return new int[]{2};
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java b/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java
deleted file mode 100644
index 6fbe4a5..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/BinaryScalar.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-/**
- * Lop to perform binary scalar operations. Both inputs must be scalars.
- * Example i = j + k, i = i + 1.
- */
-
-public class BinaryScalar extends Lop
-{
-
- public enum OperationTypes {
- ADD, SUBTRACT, SUBTRACTRIGHT, MULTIPLY, DIVIDE, MODULUS, INTDIV,
- LESS_THAN, LESS_THAN_OR_EQUALS, GREATER_THAN, GREATER_THAN_OR_EQUALS, EQUALS, NOT_EQUALS,
- AND, OR,
- LOG,POW,MAX,MIN,PRINT,
- IQSIZE,
- Over,
- SEQINCR
- }
-
- OperationTypes operation;
-
- /**
- * This overloaded constructor is used for setting exec type in case of spark backend
- */
- public BinaryScalar(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt, ExecType et)
- {
- super(Lop.Type.BinaryCP, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false; // this field does not carry any meaning for this lop
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- /**
- * Constructor to perform a scalar operation
- * @param input
- * @param op
- */
-
- public BinaryScalar(Lop input1, Lop input2, OperationTypes op, DataType dt, ValueType vt)
- {
- super(Lop.Type.BinaryCP, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false; // this field does not carry any meaning for this lop
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- this.lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- @Override
- public String toString() {
- return "Operation: " + operation;
- }
-
- public OperationTypes getOperationType(){
- return operation;
- }
-
- @Override
- public String getInstructions(String input1, String input2, String output) throws LopsException
- {
- String opString = getOpcode( operation );
-
-
-
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append( opString );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepOutputOperand(output));
-
- return sb.toString();
- }
-
- @Override
- public Lop.SimpleInstType getSimpleInstructionType()
- {
- switch (operation){
-
- default:
- return SimpleInstType.Scalar;
- }
- }
-
- /**
- *
- * @param op
- * @return
- */
- public static String getOpcode( OperationTypes op )
- {
- switch ( op )
- {
- /* Arithmetic */
- case ADD:
- return "+";
- case SUBTRACT:
- return "-";
- case MULTIPLY:
- return "*";
- case DIVIDE:
- return "/";
- case MODULUS:
- return "%%";
- case INTDIV:
- return "%/%";
- case POW:
- return "^";
-
- /* Relational */
- case LESS_THAN:
- return "<";
- case LESS_THAN_OR_EQUALS:
- return "<=";
- case GREATER_THAN:
- return ">";
- case GREATER_THAN_OR_EQUALS:
- return ">=";
- case EQUALS:
- return "==";
- case NOT_EQUALS:
- return "!=";
-
- /* Boolean */
- case AND:
- return "&&";
- case OR:
- return "||";
-
- /* Builtin Functions */
- case LOG:
- return "log";
- case MIN:
- return "min";
- case MAX:
- return "max";
-
- case PRINT:
- return "print";
-
- case IQSIZE:
- return "iqsize";
-
- case SEQINCR:
- return "seqincr";
-
- default:
- throw new UnsupportedOperationException("Instruction is not defined for BinaryScalar operator: " + op);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java b/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java
deleted file mode 100644
index d6b9171..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/BinaryUAggChain.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class BinaryUAggChain extends Lop
-{
-
- public static final String OPCODE = "binuaggchain";
-
- //outer operation
- private Binary.OperationTypes _binOp = null;
- //inner operation
- private Aggregate.OperationTypes _uaggOp = null;
- private PartialAggregate.DirectionTypes _uaggDir = null;
-
-
- /**
- * Constructor to setup a map mult chain without weights
- *
- * @param input
- * @param op
- * @return
- * @throws LopsException
- */
- public BinaryUAggChain(Lop input1, Binary.OperationTypes bop, Aggregate.OperationTypes uaop, PartialAggregate.DirectionTypes uadir, DataType dt, ValueType vt, ExecType et)
- throws LopsException
- {
- super(Lop.Type.BinUaggChain, dt, vt);
- addInput(input1); //X
- input1.addOutput(this);
-
- //setup operator types
- _binOp = bop;
- _uaggOp = uaop;
- _uaggDir = uadir;
-
- //setup MR parameters
- if( et == ExecType.MR )
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.GMR);
- lps.addCompatibility(JobType.DATAGEN);
- lps.addCompatibility(JobType.REBLOCK);
- lps.addCompatibility(JobType.CSV_REBLOCK);
- lps.addCompatibility(JobType.MMCJ);
- lps.addCompatibility(JobType.MMRJ);
- lps.setProperties( inputs, ExecType.MR, ExecLocation.MapOrReduce, breaksAlignment, aligner, definesMRJob );
- }
- else //SPARK
- {
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
-
- public String toString() {
- return "Operation = BinUaggChain";
- }
-
- @Override
- public String getInstructions(int input_index1, int output_index)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output)
- {
- StringBuilder sb = new StringBuilder();
-
- //exec type
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inst op code
- sb.append(OPCODE);
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //outer operation op code
- sb.append(Binary.getOpcode(_binOp));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inner operation op code
- sb.append(PartialAggregate.getOpcode(_uaggOp, _uaggDir));
- sb.append(Lop.OPERAND_DELIMITOR);
-
- //inputs and outputs
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java b/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java
deleted file mode 100644
index e856de6..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CSVReBlock.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin.OperationTypes;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-
-
-/**
- * Lop to convert CSV data into SystemML data format (TextCell, BinaryCell, or BinaryBlock)
- */
-public class CSVReBlock extends Lop
-{
-
- public static final String OPCODE = "csvrblk";
-
- Long rows_per_block;
- Long cols_per_block;
-
- public CSVReBlock(Lop input, Long rows_per_block, Long cols_per_block, DataType dt, ValueType vt, ExecType et) throws LopsException
- {
- super(Lop.Type.CSVReBlock, dt, vt);
- this.addInput(input);
- input.addOutput(this);
-
- this.rows_per_block = rows_per_block;
- this.cols_per_block = cols_per_block;
-
- /*
- * This lop can be executed only in CSVREBLOCK job.
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
-
- // If the input to reblock is a tranform, then piggyback it along with transform
- if ( input instanceof ParameterizedBuiltin
- && ((ParameterizedBuiltin)input).getOp() == OperationTypes.TRANSFORM )
- {
- definesMRJob = false;
- lps.addCompatibility(JobType.TRANSFORM);
- }
- else
- {
- lps.addCompatibility(JobType.CSV_REBLOCK);
- }
-
- if(et == ExecType.MR) {
- this.lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- }
- else if(et == ExecType.SPARK) {
- this.lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
- else {
- throw new LopsException("Incorrect execution type for CSVReblock:" + et);
- }
- }
-
- @Override
- public String toString() {
-
- return "CSVReblock - rows per block = " + rows_per_block + " cols per block " + cols_per_block ;
- }
-
- private String prepCSVProperties() throws LopsException {
- StringBuilder sb = new StringBuilder();
-
- boolean isSparkTransformInput = false;
- Data dataInput = null;
- if(getInputs().get(0).getType() == Type.Data)
- dataInput = (Data)getInputs().get(0);
- else if ( getInputs().get(0).getType() == Type.ParameterizedBuiltin && ((ParameterizedBuiltin)getInputs().get(0)).getOp() == OperationTypes.TRANSFORM) {
- isSparkTransformInput = (getExecType() == ExecType.SPARK);
- dataInput = (Data) ((ParameterizedBuiltin)getInputs().get(0)).getNamedInput(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_DATA);
- }
-
- Lop headerLop = dataInput.getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
- Lop delimLop = dataInput.getNamedInputLop(DataExpression.DELIM_DELIMITER);
- Lop fillLop = dataInput.getNamedInputLop(DataExpression.DELIM_FILL);
- Lop fillValueLop = dataInput.getNamedInputLop(DataExpression.DELIM_FILL_VALUE);
-
- if (headerLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_HAS_HEADER_ROW
- + " must be a literal.");
- if (delimLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_DELIMITER
- + " must be a literal.");
- if (fillLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_FILL
- + " must be a literal.");
- if (fillValueLop.isVariable())
- throw new LopsException(this.printErrorLocation()
- + "Parameter " + DataExpression.DELIM_FILL_VALUE
- + " must be a literal.");
-
- // Output from transform() does not have a header
- // On MR, reblock is piggybacked along with transform, and hence
- // specific information about header needn't be passed through instruction
- sb.append( ((Data)headerLop).getBooleanValue() && !isSparkTransformInput );
- sb.append( OPERAND_DELIMITOR );
- sb.append( ((Data)delimLop).getStringValue() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( ((Data)fillLop).getBooleanValue() );
- sb.append( OPERAND_DELIMITOR );
- sb.append( ((Data)fillValueLop).getDoubleValue() );
-
- return sb.toString();
-}
-
- @Override
- public String getInstructions(int input_index, int output_index) throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
-
- Lop input = getInputs().get(0);
-
- sb.append( input.prepInputOperand(input_index) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( rows_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append( cols_per_block );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepCSVProperties() );
-
- return sb.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output) throws LopsException {
- if(getExecType() != ExecType.SPARK) {
- throw new LopsException("The method getInstructions(String,String) for CSVReblock should be called only for Spark execution type");
- }
-
- if (this.getInputs().size() == 1) {
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( this.prepOutputOperand(output));
- sb.append( OPERAND_DELIMITOR );
- sb.append( rows_per_block );
- sb.append( OPERAND_DELIMITOR );
- sb.append( cols_per_block );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepCSVProperties() );
-
- return sb.toString();
-
- } else {
- throw new LopsException(this.printErrorLocation() + "Invalid number of operands ("
- + this.getInputs().size() + ") for CSVReblock operation");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java b/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java
deleted file mode 100644
index b7ab685..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CentralMoment.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform cross product operation
- */
-public class CentralMoment extends Lop
-{
-
- /**
- * Constructor to perform central moment.
- * input1 <- data (weighted or unweighted)
- * input2 <- order (integer: 0, 2, 3, or 4)
- */
-
- private void init(Lop input1, Lop input2, Lop input3, ExecType et) {
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
- if ( et == ExecType.MR ) {
- definesMRJob = true;
- lps.addCompatibility(JobType.CM_COV);
- lps.setProperties(inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- }
- else { //CP/SPARK
- // when executing in CP, this lop takes an optional 3rd input (Weights)
- if ( input3 != null ) {
- this.addInput(input3);
- input3.addOutput(this);
- }
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- public CentralMoment(Lop input1, Lop input2, DataType dt, ValueType vt) {
- this(input1, input2, null, dt, vt, ExecType.MR);
- }
-
- public CentralMoment(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et) {
- this(input1, input2, null, dt, vt, et);
- }
-
- public CentralMoment(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt) {
- this(input1, input2, input3, dt, vt, ExecType.MR);
- }
-
- public CentralMoment(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et) {
- super(Lop.Type.CentralMoment, dt, vt);
- init(input1, input2, input3, et);
- }
-
- @Override
- public String toString() {
-
- return "Operation = CentralMoment";
- }
-
- /**
- * Function to generate CP centralMoment instruction for weighted operation.
- *
- * input1: data
- * input2: weights
- * input3: order
- */
- @Override
- public String getInstructions(String input1, String input2, String input3, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( "cm" );
- sb.append( OPERAND_DELIMITOR );
-
- // Input data
- sb.append( getInputs().get(0).prepInputOperand(input1) );
- sb.append( OPERAND_DELIMITOR );
-
- // Weights
- sb.append( getInputs().get(1).prepInputOperand(input2) );
- sb.append( OPERAND_DELIMITOR );
-
- // Order
- sb.append( getInputs().get(2).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function to generate CP centralMoment instruction for unweighted operation.
- *
- * input1: data
- * input2: order (not used, and order is derived internally!)
- */
- @Override
- public String getInstructions(String input1, String input2, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
-
- sb.append( "cm" );
- sb.append( OPERAND_DELIMITOR );
-
- // Input data (can be weighted or unweighted)
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- // Order
- sb.append( getInputs().get(1).prepScalarInputOperand(getExecType()) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function to generate MR central moment instruction, invoked from
- * <code>Dag.java:getAggAndOtherInstructions()</code>. This function
- * is used for both weighted and unweighted operations.
- *
- * input_index: data (in case of weighted: data combined via combinebinary)
- * output_index: produced output
- *
- * The order for central moment is derived internally in the function.
- */
- @Override
- public String getInstructions(int input_index, int output_index) {
- return getInstructions(input_index+"", "", output_index+"");
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java b/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java
deleted file mode 100644
index e74699b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Checkpoint.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import org.apache.spark.storage.StorageLevel;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop for checkpoint operations. For example, on Spark, the semantic of a checkpoint
- * is to persist an intermediate result into a specific storage level (e.g., mem_only).
- *
- * We use the name checkpoint in terms of cache/persist in Spark (not in terms of checkpoint
- * in Spark streaming) in order to differentiate from CP caching.
- *
- * NOTE: since this class uses spark apis, it should only be instantiated if we are
- * running in execution mode spark (whenever all spark libraries are available)
- */
-public class Checkpoint extends Lop
-{
- public static final String OPCODE = "chkpoint";
-
- public static final StorageLevel DEFAULT_STORAGE_LEVEL = StorageLevel.MEMORY_AND_DISK();
- public static final StorageLevel SER_STORAGE_LEVEL = StorageLevel.MEMORY_AND_DISK_SER();
- public static final String STORAGE_LEVEL = "storage.level";
-
- private StorageLevel _storageLevel;
-
-
- /**
- * TODO change string parameter storage.level to StorageLevel as soon as we can assume
- * that Spark libraries are always available.
- *
- * @param input
- * @param dt
- * @param vt
- * @param level
- * @param et
- * @throws LopsException
- */
- public Checkpoint(Lop input, DataType dt, ValueType vt, String level)
- throws LopsException
- {
- super(Lop.Type.Checkpoint, dt, vt);
- this.addInput(input);
- input.addOutput(this);
-
- _storageLevel = StorageLevel.fromString(level);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = false;
-
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
- }
-
- public StorageLevel getStorageLevel()
- {
- return _storageLevel;
- }
-
- public void setStorageLevel(StorageLevel level)
- {
- _storageLevel = level;
- }
-
- @Override
- public String toString() {
- return "Checkpoint - storage.level = " + _storageLevel.toString();
- }
-
- @Override
- public String getInstructions(String input1, String output)
- throws LopsException
- {
- //valid execution type
- if(getExecType() != ExecType.SPARK) {
- throw new LopsException("Wrong execution type for Checkpoint.getInstructions (expected: SPARK, found: "+getExecType()+").");
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( OPCODE );
- sb.append( OPERAND_DELIMITOR );
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
- sb.append( prepOutputOperand(output));
- sb.append( OPERAND_DELIMITOR );
- sb.append( getStorageLevelString(_storageLevel) );
-
- return sb.toString();
-
- }
-
- /**
- * This is a utility method because Sparks StorageLevel.toString() is incompatible with its own
- * fromString() method.
- *
- * @param level
- * @return
- */
- public static String getStorageLevelString( StorageLevel level)
- {
- if( StorageLevel.NONE().equals(level) )
- return "NONE";
- else if( StorageLevel.MEMORY_ONLY().equals(level) )
- return "MEMORY_ONLY";
- else if( StorageLevel.MEMORY_ONLY_2().equals(level) )
- return "MEMORY_ONLY_2";
- else if( StorageLevel.MEMORY_ONLY_SER().equals(level) )
- return "MEMORY_ONLY_SER";
- else if( StorageLevel.MEMORY_ONLY_SER_2().equals(level) )
- return "MEMORY_ONLY_SER_2";
- else if( StorageLevel.MEMORY_AND_DISK().equals(level) )
- return "MEMORY_AND_DISK";
- else if( StorageLevel.MEMORY_AND_DISK_2().equals(level) )
- return "MEMORY_AND_DISK_2";
- else if( StorageLevel.MEMORY_AND_DISK_SER().equals(level) )
- return "MEMORY_AND_DISK_SER";
- else if( StorageLevel.MEMORY_AND_DISK_SER_2().equals(level) )
- return "MEMORY_AND_DISK_SER_2";
- else if( StorageLevel.DISK_ONLY().equals(level) )
- return "DISK_ONLY";
- else if( StorageLevel.DISK_ONLY_2().equals(level) )
- return "DISK_ONLY_2";
-
- return "INVALID";
- }
-
- /**
- *
- * @return
- */
- public static String getDefaultStorageLevelString() {
- return getStorageLevelString( DEFAULT_STORAGE_LEVEL );
- }
-
- /**
- *
- * @return
- */
- public static String getSerializeStorageLevelString() {
- return getStorageLevelString( SER_STORAGE_LEVEL );
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CoVariance.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CoVariance.java b/src/main/java/com/ibm/bi/dml/lops/CoVariance.java
deleted file mode 100644
index 2d1626e..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CoVariance.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- * Lop to compute covariance between two 1D matrices
- *
- */
-public class CoVariance extends Lop
-{
-
-
- /**
- * Constructor to perform covariance.
- * input1 <- data
- * (prior to this lop, input vectors need to attached together using CombineBinary or CombineTertiary)
- * @throws LopsException
- */
-
- public CoVariance(Lop input1, DataType dt, ValueType vt) throws LopsException {
- this(input1, dt, vt, ExecType.MR);
- }
-
- public CoVariance(Lop input1, DataType dt, ValueType vt, ExecType et) throws LopsException {
- super(Lop.Type.CoVariance, dt, vt);
- init(input1, null, null, et);
- }
-
- public CoVariance(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et) throws LopsException {
- this(input1, input2, null, dt, vt, et);
- }
-
- public CoVariance(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et) throws LopsException {
- super(Lop.Type.CoVariance, dt, vt);
- init(input1, input2, input3, et);
- }
-
- private void init(Lop input1, Lop input2, Lop input3, ExecType et)
- throws LopsException
- {
- /*
- * When et = MR: covariance lop will have a single input lop, which
- * denote the combined input data -- output of combinebinary, if unweighed;
- * and output combineteriaty (if weighted).
- *
- * When et = CP: covariance lop must have at least two input lops, which
- * denote the two input columns on which covariance is computed. It also
- * takes an optional third arguments, when weighted covariance is computed.
- */
- addInput(input1);
- input1.addOutput(this);
-
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
- if ( et == ExecType.MR )
- {
- lps.addCompatibility(JobType.CM_COV);
- lps.setProperties(inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- }
- else //CP/SPARK
- {
- definesMRJob = false;
- if ( input2 == null ) {
- throw new LopsException(this.printErrorLocation() + "Invalid inputs to covariance lop.");
- }
- addInput(input2);
- input2.addOutput(this);
-
- if ( input3 != null ) {
- addInput(input3);
- input3.addOutput(this);
- }
- lps.addCompatibility(JobType.INVALID);
- lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
- }
- }
-
- @Override
- public String toString() {
-
- return "Operation = coVariance";
- }
-
- /**
- * Function two generate CP instruction to compute unweighted covariance.
- * input1 -> input column 1
- * input2 -> input column 2
- */
- @Override
- public String getInstructions(String input1, String input2, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cov" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function two generate CP instruction to compute weighted covariance.
- * input1 -> input column 1
- * input2 -> input column 2
- * input3 -> weights
- */
- @Override
- public String getInstructions(String input1, String input2, String input3, String output) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cov" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepInputOperand(input3));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output));
-
- return sb.toString();
- }
-
- /**
- * Function to generate MR version of covariance instruction.
- * input_index -> denote the "combined" input columns and weights,
- * when applicable.
- */
- @Override
- public String getInstructions(int input_index, int output_index) {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "cov" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append ( this.prepInputOperand(output_index));
-
- return sb.toString();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java b/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java
deleted file mode 100644
index 81fafef..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CombineBinary.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to represent an combine operation -- used ONLY in the context of sort.
- */
-
-public class CombineBinary extends Lop
-{
-
-
- public enum OperationTypes {PreSort, PreCentralMoment, PreCovUnweighted, PreGroupedAggUnweighted}; // (PreCovWeighted,PreGroupedAggWeighted) will be CombineTertiary
- OperationTypes operation;
-
- /**
- * @param input - input lop
- * @param op - operation type
- */
-
- public CombineBinary(OperationTypes op, Lop input1, Lop input2, DataType dt, ValueType vt)
- {
- super(Lop.Type.CombineBinary, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- input1.addOutput(this);
- input2.addOutput(this);
-
- /*
- * This lop can ONLY be executed as a STANDALONE job
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
- lps.addCompatibility(JobType.COMBINE);
- this.lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
- this.lps.setProducesIntermediateOutput(true);
- }
-
- public String toString()
- {
- return "combinebinary";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2, int output_index)
- throws LopsException
- {
- // Determine whether or not the second input denotes weights vector.
- // CombineBinary can be used to combine (data,weights) vectors or (data1,data2) vectors
- boolean isSecondInputIsWeight = true;
- if ( operation == OperationTypes.PreCovUnweighted || operation == OperationTypes.PreGroupedAggUnweighted ) {
- isSecondInputIsWeight = false;
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "combinebinary" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( prepOperand(String.valueOf(isSecondInputIsWeight), DataType.SCALAR, ValueType.BOOLEAN) );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepOutputOperand(output_index));
-
- return sb.toString();
- }
-
- public OperationTypes getOperation() {
- return operation;
- }
-
- public static CombineBinary constructCombineLop(OperationTypes op, Lop input1,
- Lop input2, DataType dt, ValueType vt) {
-
- HashSet<Lop> set1 = new HashSet<Lop>();
- set1.addAll(input1.getOutputs());
-
- // find intersection of input1.getOutputs() and input2.getOutputs();
- set1.retainAll(input2.getOutputs());
-
- for (Lop lop : set1) {
- if ( lop.type == Lop.Type.CombineBinary ) {
- CombineBinary combine = (CombineBinary)lop;
- if ( combine.operation == op)
- return (CombineBinary)lop;
- }
- }
-
- CombineBinary comn = new CombineBinary(op, input1, input2, dt, vt);
- comn.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return comn;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java b/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java
deleted file mode 100644
index 529d57b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/CombineTernary.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashSet;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to represent an combine operation -- used ONLY in the context of sort.
- */
-
-public class CombineTernary extends Lop
-{
-
- public enum OperationTypes {
- PreCovWeighted, PreGroupedAggWeighted
- }; // PreCovUnweighted,PreGroupedAggWeighted will be CombineBinary
-
- OperationTypes operation;
-
- public CombineTernary( OperationTypes op, Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt) {
- super(Lop.Type.CombineTernary, dt, vt);
- operation = op;
- this.addInput(input1);
- this.addInput(input2);
- this.addInput(input3);
- input1.addOutput(this);
- input2.addOutput(this);
- input3.addOutput(this);
-
- /*
- * This lop can ONLY be executed as a STANDALONE job
- */
- boolean breaksAlignment = false;
- boolean aligner = false;
- boolean definesMRJob = true;
- this.lps.addCompatibility(JobType.COMBINE);
- this.lps.setProperties(inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
- this.lps.setProducesIntermediateOutput(true);
- }
-
- public String toString() {
- return "combineternary";
- }
-
- @Override
- public String getInstructions(int input_index1, int input_index2,
- int input_index3, int output_index)
- throws LopsException
- {
- StringBuilder sb = new StringBuilder();
- sb.append( getExecType() );
- sb.append( Lop.OPERAND_DELIMITOR );
- sb.append( "combineternary" );
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(0).prepInputOperand(input_index1));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(1).prepInputOperand(input_index2));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( getInputs().get(2).prepInputOperand(input_index3));
- sb.append( OPERAND_DELIMITOR );
-
- sb.append( this.prepInputOperand(output_index));
-
- return sb.toString();
- }
-
- public OperationTypes getOperation() {
- return operation;
- }
-
- public static CombineTernary constructCombineLop( OperationTypes op, Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt) {
-
- HashSet<Lop> set1 = new HashSet<Lop>();
- set1.addAll(input1.getOutputs());
-
- // find intersection of input1.getOutputs() and input2.getOutputs()
- set1.retainAll(input2.getOutputs());
-
- // find intersection of the above result and input3.getOutputs()
- set1.retainAll(input3.getOutputs());
-
- for (Lop lop : set1) {
- if (lop.type == Lop.Type.CombineTernary) {
- CombineTernary combine = (CombineTernary) lop;
- if (combine.operation == op)
- return (CombineTernary) lop;
- }
- }
-
- CombineTernary comn = new CombineTernary(op, input1, input2, input3, dt, vt);
- comn.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return comn;
- }
-
-}