[27/78] [abbrv] [partial] incubator-systemml git commit: Move files to new package folder structure

[lr...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/Lop.java b/src/main/java/com/ibm/bi/dml/lops/Lop.java
deleted file mode 100644
index 751ca1f..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/Lop.java
+++ /dev/null
@@ -1,720 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Base class for all Lops.
- */
-
-public abstract class Lop 
-{
-	
-	public enum Type {
-		Data, DataGen, 										//CP/MR read/write/datagen 
-		ReBlock, CSVReBlock,								//MR reblock operations
-		MMCJ, MMRJ, MMTSJ, PMMJ, MapMult, MapMultChain,     //MR matrix multiplications
-		UnaryCP, UNARY, BinaryCP, Binary, Ternary,          //CP/MR unary/binary/ternary
-		RangeReIndex, LeftIndex, ZeroOut,                   //CP/MR indexing 
-		Aggregate, PartialAggregate,   	   				    //CP/MR aggregation
-		BinUaggChain, UaggOuterChain,  	                    //CP/MR aggregation
-		TernaryAggregate,                                   //CP ternary-binary aggregates
-		Grouping, 											//MR grouping
-		Append,                                             //CP/MR append (column append)
-		CombineUnary, CombineBinary, CombineTernary,        //MR combine (stitch together)
-		CentralMoment, CoVariance, GroupedAgg,
-		Transform, DataPartition, RepMat,                   //CP/MR reorganization, partitioning, replication
-		ParameterizedBuiltin,                               //CP/MR parameterized ops (name/value)
-		FunctionCallCP, 									//CP function calls 
-		CumulativePartialAggregate, CumulativeSplitAggregate, CumulativeOffsetBinary, //MR cumsum/cumprod/cummin/cummax
-		WeightedSquaredLoss, WeightedSigmoid, WeightedDivMM, WeightedCeMM, WeightedUMM,
-		SortKeys, PickValues,
-		Checkpoint, 										//Spark persist into storage level
-	};
-
-	/**
-	 * Lop types
-	 */
-	public enum SimpleInstType {
-		Scalar, Variable, File
-	};
-
-	public enum VisitStatus {
-		DONE, VISITING, NOTVISITED
-	}
-	
-
-	protected static final Log LOG =  LogFactory.getLog(Lop.class.getName());
-	
-	public static final String FILE_SEPARATOR = "/";
-	public static final String PROCESS_PREFIX = "_p";
-	
-	public static final String INSTRUCTION_DELIMITOR = "\u2021"; // "\u002c"; //",";
-	public static final String OPERAND_DELIMITOR = "\u00b0"; //\u2021"; //00ea"; //"::#::";
-	public static final String VALUETYPE_PREFIX = "\u00b7" ; //":#:";
-	public static final String DATATYPE_PREFIX = VALUETYPE_PREFIX; //":#:";
-	public static final String LITERAL_PREFIX = VALUETYPE_PREFIX; //":#:";
-	public static final String NAME_VALUE_SEPARATOR = "="; // e.g., used in parameterized builtins
-	public static final String VARIABLE_NAME_PLACEHOLDER = "##"; //TODO: use in LOPs 
-	public static final String MATRIX_VAR_NAME_PREFIX = "_mVar";
-	public static final String SCALAR_VAR_NAME_PREFIX = "_Var";
-	
-	// Boolean array to hold the list of nodes(lops) in the DAG that are reachable from this lop.
-	private boolean[] reachable = null;
-	private DataType _dataType;
-	private ValueType _valueType;
-
-	private VisitStatus _visited = VisitStatus.NOTVISITED;
-
-	protected Lop.Type type;
-	
-	/**
-	 * transient indicator
-	 */
-	protected boolean hasTransientParameters = false;
-
-	/**
-	 * handle to all inputs and outputs.
-	 */
-	protected ArrayList<Lop> inputs;
-	protected ArrayList<Lop> outputs;
-	
-	/**
-	 * refers to #lops whose input is equal to the output produced by this lop.
-	 * This is used in generating rmvar instructions as soon as the output produced
-	 * by this lop is consumed. Otherwise, such rmvar instructions are added 
-	 * at the end of program blocks. 
-	 * 
-	 */
-	protected int consumerCount;
-
-	/**
-	 * handle to output parameters, dimensions, blocking, etc.
-	 */
-
-	protected OutputParameters outParams = null;
-
-	protected LopProperties lps = null;
-	
-
-	/**
-	 * Constructor to be invoked by base class.
-	 * 
-	 * @param t
-	 */
-
-	public Lop(Type t, DataType dt, ValueType vt) {
-		type = t;
-		_dataType = dt; // data type of the output produced from this LOP
-		_valueType = vt; // value type of the output produced from this LOP
-		inputs = new ArrayList<Lop>();
-		outputs = new ArrayList<Lop>();
-		outParams = new OutputParameters();
-		lps = new LopProperties();
-	}
-	
-	/**
-	 * get visit status of node
-	 * 
-	 * @return
-	 */
-
-	public VisitStatus getVisited() {
-		return _visited;
-	}
-
-	/**
-	 * set visit status of node
-	 * 
-	 * @param visited
-	 */
-	public void setVisited(VisitStatus visited) {
-		_visited = visited;
-	}
-
-	
-	public boolean[] get_reachable() {
-		return reachable;
-	}
-
-	public boolean[] create_reachable(int size) {
-		reachable = new boolean[size];
-		return reachable;
-	}
-
-	/**
-	 * get data type of the output that is produced by this lop
-	 * 
-	 * @return
-	 */
-
-	public DataType getDataType() {
-		return _dataType;
-	}
-
-	/**
-	 * set data type of the output that is produced by this lop
-	 * 
-	 * @param dt
-	 */
-	public void setDataType(DataType dt) {
-		_dataType = dt;
-	}
-
-	/**
-	 * get value type of the output that is produced by this lop
-	 * 
-	 * @return
-	 */
-
-	public ValueType getValueType() {
-		return _valueType;
-	}
-
-	/**
-	 * set value type of the output that is produced by this lop
-	 * 
-	 * @param vt
-	 */
-	public void setValueType(ValueType vt) {
-		_valueType = vt;
-	}
-
-
-	/**
-	 * Method to get Lop type.
-	 * 
-	 * @return
-	 */
-
-	public Lop.Type getType() {
-		return type;
-	}
-
-	/**
-	 * Method to get input of Lops
-	 * 
-	 * @return
-	 */
-	public ArrayList<Lop> getInputs() {
-		return inputs;
-	}
-
-	/**
-	 * Method to get output of Lops
-	 * 
-	 * @return
-	 */
-
-	public ArrayList<Lop> getOutputs() {
-		return outputs;
-	}
-
-	/**
-	 * Method to add input to Lop
-	 * 
-	 * @param op
-	 */
-
-	public void addInput(Lop op) {
-		inputs.add(op);
-	}
-
-	/**
-	 * Method to add output to Lop
-	 * 
-	 * @param op
-	 */
-
-	public void addOutput(Lop op) {
-		outputs.add(op);
-	}
-	
-	public int getConsumerCount() {
-		return consumerCount;
-	}
-	
-	public void setConsumerCount(int cc) {
-		consumerCount = cc;
-	}
-	
-	public int removeConsumer() {
-		consumerCount--;
-		return consumerCount;
-	}
-
-	/**
-	 * Method to have Lops print their state. This is for debugging purposes.
-	 */
-
-	public abstract String toString();
-
-	public void resetVisitStatus() {
-		if (this.getVisited() == Lop.VisitStatus.NOTVISITED)
-			return;
-		for (int i = 0; i < this.getInputs().size(); i++) {
-			this.getInputs().get(i).resetVisitStatus();
-		}
-		this.setVisited(Lop.VisitStatus.NOTVISITED);
-	}
-
-	/**
-	 * Method to have recursively print state of Lop graph.
-	 */
-
-	public final void printMe() {
-		if (LOG.isDebugEnabled()){
-			StringBuilder s = new StringBuilder("");
-			if (this.getVisited() != VisitStatus.DONE) {
-				s.append(getType() + ": " + getID() + "\n" ); // hashCode());
-				s.append("Inputs: ");
-				for (int i = 0; i < this.getInputs().size(); i++) {
-					s.append(" " + this.getInputs().get(i).getID() + " ");
-				}
-
-				s.append("\n");
-				s.append("Outputs: ");
-				for (int i = 0; i < this.getOutputs().size(); i++) {
-					s.append(" " + this.getOutputs().get(i).getID() + " ");
-				}
-
-				s.append("\n");
-				s.append(this.toString());
-				s.append("Begin Line: " + _beginLine + ", Begin Column: " + _beginColumn + ", End Line: " + _endLine + ", End Column: " + _endColumn + "\n");
-				s.append("FORMAT:" + this.getOutputParameters().getFormat() + ", rows="
-						+ this.getOutputParameters().getNumRows() + ", cols=" + this.getOutputParameters().getNumCols()
-						+ ", Blocked?: " + this.getOutputParameters().isBlocked() + ", rowsInBlock=" + 
-						this.getOutputParameters().getRowsInBlock() + ", colsInBlock=" + 
-						this.getOutputParameters().getColsInBlock() + "\n");
-				this.setVisited(VisitStatus.DONE);
-				s.append("\n");
-
-				for (int i = 0; i < this.getInputs().size(); i++) {
-					this.getInputs().get(i).printMe();
-				}
-			}
-			LOG.debug(s.toString());
-		}
-	}
-
-	/**
-	 * Method to return the ID of LOP
-	 */
-	public long getID() {
-		return lps.getID();
-	}
-	
-	public int getLevel() {
-		return lps.getLevel();
-	}
-	
-	public void setLevel() {
-		lps.setLevel(inputs);
-	}
-	
-	/**
-	 * Method to get the location property of LOP
-	 * 
-	 * @return
-	 */
- 	public ExecLocation getExecLocation() {
-		return lps.getExecLocation();
-	}
- 
-	/**
-	 * Method to get the execution type (CP or MR) of LOP
-	 * 
-	 * @return
-	 */
- 	public ExecType getExecType() {
-		return lps.getExecType();
-	}
- 
-	/**
-	 * Method to get the compatible job type for the LOP
-	 * @return
-	 */
-	
-	public int getCompatibleJobs() {
-		return lps.getCompatibleJobs();
-	}
-	
-	/**
-	 * Method to find if the lop breaks alignment
-	 */
-	public boolean getBreaksAlignment() {
-		return lps.getBreaksAlignment();
-	}
-	
-	public boolean getProducesIntermediateOutput() {
-		return lps.getProducesIntermediateOutput();
-	}
-	
-	public boolean isAligner()
-	{
-		return lps.isAligner();
-	}
-
-	public boolean definesMRJob()
-	{
-		return lps.getDefinesMRJob();
-	}
-
-	/**
-	 * Method to recursively add LOPS to a DAG
-	 * 
-	 * @param dag
-	 */
-	public final void addToDag(Dag<Lop> dag) 
-	{
-		if( dag.addNode(this) )
-			for( Lop l : getInputs() )
-				l.addToDag(dag);
-	}
-
-	/**
-	 * Method to get output parameters
-	 * 
-	 * @return
-	 */
-
-	public OutputParameters getOutputParameters() {
-		return outParams;
-	}
-	
-	/** Method should be overridden if needed **/
-	public String getInstructions(String input1, String input2, String input3, String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-	
-	/** Method should be overridden if needed **/
-	public String getInstructions(String input1, String input2, String input3, String input4, String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(String input1, String input2, String input3, String input4, String input5, String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(String input1, String input2, String input3, String input4, String input5, String input6, String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-	
-	public String getInstructions(int output_index) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass. Lop Type: " + this.getType());
-	}
-
-	public String getInstructions(int input_index, int output_index) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass. Lop Type: " + this.getType());
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(int input_index1, int input_index2, int output_index) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-	
-	/** Method should be overridden if needed **/
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(String input1, String input2, String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(String input1, String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public String getInstructions(String[] inputs, String[] outputs) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-	
-	/** Method should be overridden if needed **/
-	public String getInstructions() throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/** Method should be overridden if needed **/
-	public SimpleInstType getSimpleInstructionType() throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	///////////////////////////////////////////////////////////////////////////
-	// store position information for Lops
-	///////////////////////////////////////////////////////////////////////////
-	public int _beginLine, _beginColumn;
-	public int _endLine, _endColumn;
-	
-	public void setBeginLine(int passed)    { _beginLine = passed;   }
-	public void setBeginColumn(int passed) 	{ _beginColumn = passed; }
-	public void setEndLine(int passed) 		{ _endLine = passed;   }
-	public void setEndColumn(int passed)	{ _endColumn = passed; }
-	
-	public void setAllPositions(int blp, int bcp, int elp, int ecp){
-		_beginLine	 = blp; 
-		_beginColumn = bcp; 
-		_endLine 	 = elp;
-		_endColumn 	 = ecp;
-	}
-
-	public int getBeginLine()	{ return _beginLine;   }
-	public int getBeginColumn() { return _beginColumn; }
-	public int getEndLine() 	{ return _endLine;   }
-	public int getEndColumn()	{ return _endColumn; }
-	
-	public String printErrorLocation(){
-		return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
-	}
-	
-	public String printWarningLocation(){
-		return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
-	}
-
-	//TODO: Leo This might get confused with Rand.getInstructions
-	public String getInstructions(String input, String rowl, String rowu,
-			String coll, String colu, String leftRowDim,
-			String leftColDim, String output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-	
-	public String getInstructions(int input, int rowl, int rowu,
-			int coll, int colu, int leftRowDim,
-			int leftColDim, int output) throws LopsException {
-		throw new LopsException(this.printErrorLocation() + "Should never be invoked in Baseclass");
-	}
-
-	/**
-	 * Function that determines if the output of a LOP is defined by a variable or not.
-	 * 
-	 * @return
-	 */
-	public boolean isVariable() {
-		return ( (getExecLocation() == ExecLocation.Data && !((Data)this).isLiteral()) 
-				 || !(getExecLocation() == ExecLocation.Data ) );
-	}
-	
-	
-	
-	/**
-	 * Method to prepare instruction operand with given parameters.
-	 * 
-	 * @param label
-	 * @param dt
-	 * @param vt
-	 * @return
-	 */
-	public String prepOperand(String label, DataType dt, ValueType vt) {
-		StringBuilder sb = new StringBuilder();
-		sb.append(label);
-		sb.append(Lop.DATATYPE_PREFIX);
-		sb.append(dt);
-		sb.append(Lop.VALUETYPE_PREFIX);
-		sb.append(vt);
-		return sb.toString();
-	}
-
-	/**
-	 * Method to prepare instruction operand with given parameters.
-	 * 
-	 * @param label
-	 * @param dt
-	 * @param vt
-	 * @return
-	 */
-	public String prepOperand(String label, DataType dt, ValueType vt, boolean literal) {
-		StringBuilder sb = new StringBuilder();
-		sb.append(label);
-		sb.append(Lop.DATATYPE_PREFIX);
-		sb.append(dt);
-		sb.append(Lop.VALUETYPE_PREFIX);
-		sb.append(vt);
-		sb.append(Lop.LITERAL_PREFIX);
-		sb.append(literal);
-		return sb.toString();
-	}
-
-	/**
-	 * Method to prepare instruction operand with given label. Data type
-	 * and Value type are derived from Lop's properties.
-	 * 
-	 * @param label
-	 * @return
-	 */
-	private String prepOperand(String label) {
-		StringBuilder sb = new StringBuilder("");
-		sb.append(label);
-		sb.append(Lop.DATATYPE_PREFIX);
-		sb.append(getDataType());
-		sb.append(Lop.VALUETYPE_PREFIX);
-		sb.append(getValueType());
-		return sb.toString();
-	}
-	
-	public String prepOutputOperand() {
-		return prepOperand(getOutputParameters().getLabel());
-	}
-	
-	public String prepOutputOperand(int index) {
-		return prepOperand(index+"");
-	}
-	public String prepOutputOperand(String label) {
-		return prepOperand(label);
-	}
-	
-	/**
-	 * Function to prepare label for scalar inputs while generating instructions.
-	 * It attaches placeholder suffix and prefixes if the Lop denotes a variable.
-	 * 
-	 * @return
-	 */
-	public String prepScalarLabel() {
-		String ret = getOutputParameters().getLabel();
-		if ( isVariable() ){
-			ret = Lop.VARIABLE_NAME_PLACEHOLDER + ret + Lop.VARIABLE_NAME_PLACEHOLDER;
-		}
-		return ret;
-	}
-	
-	/**
-	 * Function to be used in creating instructions for creating scalar
-	 * operands. It decides whether or not attach placeholders for instruction
-	 * patching. Resulting string also encodes if the operand is a literal.
-	 * 
-	 * For non-literals: 
-	 * Placeholder prefix and suffix need to be attached for Instruction 
-	 * Patching during execution. However, they should NOT be attached IF: 
-	 *   - the operand is a literal 
-	 *     OR 
-	 *   - the execution type is CP. This is because CP runtime has access 
-	 *     to symbol table and the instruction encodes sufficient information
-	 *     to determine if an operand is a literal or not.
-	 * 
-	 * @param et
-	 * @return
-	 */
-	public String prepScalarOperand(ExecType et, String label) {
-		boolean isData = (getExecLocation() == ExecLocation.Data);
-		boolean isLiteral = (isData && ((Data)this).isLiteral());
-		
-		StringBuilder sb = new StringBuilder("");
-		if ( et == ExecType.CP || et == ExecType.SPARK || (isData && isLiteral)) {
-			sb.append(label);
-		}
-		else {
-			sb.append(Lop.VARIABLE_NAME_PLACEHOLDER);
-			sb.append(label);
-			sb.append(Lop.VARIABLE_NAME_PLACEHOLDER);
-		}
-		
-		sb.append(Lop.DATATYPE_PREFIX);
-		sb.append(getDataType());
-		sb.append(Lop.VALUETYPE_PREFIX);
-		sb.append(getValueType());
-		sb.append(Lop.LITERAL_PREFIX);
-		sb.append(isLiteral);
-		
-		return sb.toString();
-	}
-
-	public String prepScalarInputOperand(ExecType et) {
-		return prepScalarOperand(et, getOutputParameters().getLabel());
-	}
-	
-	public String prepScalarInputOperand(String label) {
-		boolean isData = (getExecLocation() == ExecLocation.Data);
-		boolean isLiteral = (isData && ((Data)this).isLiteral());
-		
-		StringBuilder sb = new StringBuilder("");
-		sb.append(label);
-		sb.append(Lop.DATATYPE_PREFIX);
-		sb.append(getDataType());
-		sb.append(Lop.VALUETYPE_PREFIX);
-		sb.append(getValueType());
-		sb.append(Lop.LITERAL_PREFIX);
-		sb.append(isLiteral);
-		
-		return sb.toString();
-	}
-
-	public String prepInputOperand(int index) {
-		return prepInputOperand(index+"");
-	}
-
-	public String prepInputOperand(String label) {
-		DataType dt = getDataType();
-		if ( dt == DataType.MATRIX ) {
-			return prepOperand(label);
-		}
-		else {
-			return prepScalarInputOperand(label);
-		}
-	}
-	
-	/**
-	 * Method to check if a LOP expects an input from the Distributed Cache.
-	 * The method in parent class always returns <code>false</code> (default).
-	 * It must be overridden by individual LOPs that use the cache.
-	 */
-	public boolean usesDistributedCache() {
-		return false;
-	}
-	
-	public int[] distributedCacheInputIndex() {
-		return new int[]{-1};
-	}
-
-	
-	public boolean hasNonBlockedInputs() {
-		for(Lop in : getInputs())
-			if(in.getDataType() == DataType.MATRIX && !in.getOutputParameters().isBlocked())
-				return true;
-		return false;
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/LopProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/LopProperties.java b/src/main/java/com/ibm/bi/dml/lops/LopProperties.java
deleted file mode 100644
index 3268e1d..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/LopProperties.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-
-public class LopProperties 
-{
-	
-	public enum ExecType { CP, CP_FILE, MR, SPARK, INVALID };
-	public enum ExecLocation {INVALID, RecordReader, Map, MapOrReduce, MapAndReduce, Reduce, Data, ControlProgram };
-
-	// static variable to assign an unique ID to every lop that is created
-	private static IDSequence UniqueLopID = null;
-	
-	static{
-		UniqueLopID = new IDSequence();
-	}
-	
-	/** 
-	 * Execution properties for each lop.
-	 * 
-	 * execLoc  = in case execType=MR, where does this lop must run in a job (map, reduce, etc.)
-	 * compatibleJobs = list of job types in which this lop can be run. It is maintained as a bit vector.
-	 * breaksAlignment = does this lop alters the keys ?
-	 * isAligner = is this lop mainly used to reorder/sort/align the keys
-	 *   
-	 */
-	long ID;
-	int level;
-	ExecType execType;
-	ExecLocation execLoc;
-	int compatibleJobs;
-	boolean breaksAlignment;
-	boolean isAligner;
-	boolean definesMRJob;
-	boolean producesIntermediateOutput;
-	
-	public LopProperties() {
-		ID = UniqueLopID.getNextID();
-		execType = ExecType.INVALID;
-		execLoc = ExecLocation.INVALID;
-		compatibleJobs = JobType.INVALID.getBase();
-		breaksAlignment = true;
-		isAligner = false;
-		definesMRJob = false;
-		producesIntermediateOutput = false;
-	}
-	
-	public long getID() { return ID; }
-	public int getLevel() { return level; }
-	public void setLevel( int l ) { level = l; }
-	
-	public ExecLocation getExecLocation() {
-		return execLoc;
-	}
-	
-	public ExecType getExecType() {
-		return execType;
-	}
-	
-	public int getCompatibleJobs() {
-		return compatibleJobs;
-	}
-	
-	public boolean getBreaksAlignment() {
-		return breaksAlignment;
-	}
-	
-	public boolean getDefinesMRJob() {
-		return definesMRJob;
-	}
-	
-	public boolean isAligner()
-	{
-		return isAligner;
-	}
-	
-	public boolean getProducesIntermediateOutput() {
-		return producesIntermediateOutput;
-	}
-
-	public void setExecLocation(ExecLocation el) {
-		execLoc = el;
-	}
-	
-	public void addCompatibility ( JobType jt ) {
-		compatibleJobs = compatibleJobs | jt.getBase();
-	}
-	
-	public void removeCompatibility ( JobType jt ) {
-		compatibleJobs = compatibleJobs ^ jt.getBase();
-	}
-	
-	public void removeNonPiggybackableJobs() {
-		// Remove compatibility with those jobs which do not allow any "other" instructions 
-		for ( JobType jt : JobType.values()) {
-			if(jt.allowsNoOtherInstructions()) {
-				compatibleJobs = compatibleJobs ^ jt.getBase();
-			}
-		}
-	}
-	
-	public void setCompatibleJobs(int cj) {
-		compatibleJobs = cj;
-	}
-	
-	public void setDefinesMRJob(boolean dmrj) {
-		definesMRJob = dmrj;
-	}
-	
-	public void setBreaksAlignment(boolean ba) {
-		breaksAlignment = ba;
-	}
-	
-	public void setAligner(boolean align) {
-		isAligner = align;
-	}
-	
-	public void setProducesIntermediateOutput(boolean pio) {
-		producesIntermediateOutput = pio;
-	}
-	
-	/*
-	 * Function to compute the node level in the entire Lops DAG. 
-	 *   level(v) = max( levels(v.inputs) ) + 1
-	 */
-	public void setLevel(ArrayList<Lop>  inputs) {
-		int tmplevel = -1;
-		if ( inputs == null || inputs.isEmpty() )
-			tmplevel = 0;
-		else {
-			// find the max level among all inputs
-			for(Lop in : inputs) {
-				if(tmplevel < in.getLevel() ) {
-					tmplevel = in.getLevel();
-				}
-			}
-			// this.level should be one more than the max
-			tmplevel = tmplevel+1;
-		}
-		setLevel(tmplevel);
-	}
-
-	public void setProperties ( ArrayList<Lop> inputs, ExecType et, ExecLocation el, boolean ba, boolean aligner, boolean definesMR ) {
-		execType = et;
-		execLoc = el;
-		breaksAlignment = ba;
-		isAligner = aligner;
-		definesMRJob = definesMR;
-		setLevel(inputs);
-	}
-	
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/LopsException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/LopsException.java b/src/main/java/com/ibm/bi/dml/lops/LopsException.java
deleted file mode 100644
index 762a3e7..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/LopsException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.api.DMLException;
-
-public class LopsException extends DMLException 
-{
-
-	private static final long serialVersionUID = 1L;
-	
-	public LopsException(String message)
-	{
-		super(message);
-	}
-
-	public LopsException(Exception e) {
-		super(e);
-	}
-	
-	public LopsException(String message, Throwable cause) {
-	    super(message, cause);
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMCJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMCJ.java b/src/main/java/com/ibm/bi/dml/lops/MMCJ.java
deleted file mode 100644
index dc37667..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMCJ.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform cross product operation
- */
-public class MMCJ extends Lop 
-{
-
-		
-	public enum MMCJType {
-		AGG,
-		NO_AGG,
-	}
-	
-	//optional attribute for mr exec type
-	private MMCJType _type = MMCJType.AGG;
-	
-	//optional attribute for spark exec type
-	private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-		
-	/**
-	 * Constructor to perform a cross product operation.
-	 * @param et 
-	 * @param input
-	 * @param op
-	 */
-
-	public MMCJ(Lop input1, Lop input2, DataType dt, ValueType vt, MMCJType type, ExecType et) 
-	{
-		super(Lop.Type.MMCJ, dt, vt);		
-		this.addInput(input1);
-		this.addInput(input2);
-		input1.addOutput(this);
-		input2.addOutput(this);
-		
-		_type = type;
-		
-		if( et == ExecType.MR )
-		{
-			boolean breaksAlignment = true;
-			boolean aligner = false;
-			boolean definesMRJob = true;
-			lps.addCompatibility(JobType.MMCJ);
-			this.lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
-			this.lps.setProducesIntermediateOutput(true);
-		}
-		else //if( et == ExecType.SPARK )
-		{
-			boolean breaksAlignment = false;
-			boolean aligner = false;
-			boolean definesMRJob = false;
-			lps.addCompatibility(JobType.INVALID);
-			lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
-		}
-	}
-
-	/**
-	 * 
-	 * @param input1
-	 * @param input2
-	 * @param dt
-	 * @param vt
-	 * @param aggtype
-	 * @param et
-	 */
-	public MMCJ(Lop input1, Lop input2, DataType dt, ValueType vt, SparkAggType aggtype, ExecType et) 
-	{
-		this(input1, input2, dt, vt, MMCJType.NO_AGG, et);
-		
-		_aggtype = aggtype;
-	}
-	
-	
-	@Override
-	public String toString() {
-	
-		return "Operation = MMCJ";
-	}
-
-	//MR instruction generation
-	@Override
-	public String getInstructions(int input_index1, int input_index2, int output_index)
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-		sb.append( "cpmm" );
-		sb.append( OPERAND_DELIMITOR );
-		
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(1).prepInputOperand(input_index2));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( this.prepOutputOperand(output_index));
-		
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( _type );
-		
-		
-		return sb.toString();
-	}
-
-	//SPARK instruction generation
-	@Override
-	public String getInstructions(String input1, String input2, String output)
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-		sb.append( "cpmm" );
-		sb.append( OPERAND_DELIMITOR );
-		
-		sb.append( getInputs().get(0).prepInputOperand(input1) );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(1).prepInputOperand(input2) );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( this.prepOutputOperand(output) );
-		
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( _aggtype );
-				
-		return sb.toString();
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMRJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMRJ.java b/src/main/java/com/ibm/bi/dml/lops/MMRJ.java
deleted file mode 100644
index 8722526..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMRJ.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform cross product operation
- */
-public class MMRJ extends Lop 
-{
-
-		
-	/**
-	 * Constructor to perform a cross product operation.
-	 * @param input
-	 * @param op
-	 */
-
-	public MMRJ(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et) 
-	{
-		//handle inputs and outputs
-		super(Lop.Type.MMRJ, dt, vt);		
-		this.addInput(input1);
-		this.addInput(input2);
-		input1.addOutput(this);
-		input2.addOutput(this);
-		
-		//set basic lop properties based on exec type
-		if( et == ExecType.MR )
-		{
-			boolean breaksAlignment = true;
-			boolean aligner = false;
-			boolean definesMRJob = true;
-			lps.addCompatibility(JobType.MMRJ);
-			this.lps.setProperties( inputs, et, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob );
-		}
-		else //if( et == ExecType.SPARK )
-		{
-			boolean breaksAlignment = false;
-			boolean aligner = false;
-			boolean definesMRJob = false;
-			lps.addCompatibility(JobType.INVALID);
-			lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
-		}
-	}
-
-	@Override
-	public String toString() {
-	
-		return "Operation = MMRJ";
-	}
-
-	@Override
-	public String getInstructions(int input_index1, int input_index2, int output_index)
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-		sb.append( "rmm" );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(1).prepInputOperand(input_index2));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( this.prepOutputOperand(output_index));
-		
-		return sb.toString();
-	}
-
-	@Override
-	public String getInstructions(String input1, String input2, String output)
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-		sb.append( "rmm" );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(0).prepInputOperand(input1));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(1).prepInputOperand(input2));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( this.prepOutputOperand(output));
-		
-		return sb.toString();
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java b/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java
deleted file mode 100644
index d1d3053..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMTSJ.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.*;
-
-
-/**
- * Lop to perform transpose-identity operation (t(X)%*%X or X%*%t(X)),
- * used to represent CP and MR instruction but in case of MR there is
- * an additional Aggregate at the reducers.
- */
-public class MMTSJ extends Lop 
-{
-
-	
-	public enum MMTSJType {
-		NONE,
-		LEFT,
-		RIGHT;
-		
-		public boolean isLeft(){
-			return (this == LEFT);
-		}
-	}
-	
-	private MMTSJType _type = null;
-	private int _numThreads = 1;
-
-	public MMTSJ(Lop input1, DataType dt, ValueType vt, ExecType et, MMTSJType type) 
-	{
-		this(input1, dt, vt, et, type, -1);
-	}
-	
-	public MMTSJ(Lop input1, DataType dt, ValueType vt, ExecType et, MMTSJType type, int k) 
-	{
-		super(Lop.Type.MMTSJ, dt, vt);		
-		addInput(input1);
-		input1.addOutput(this);
-		_type = type;
-		_numThreads = k;
-		 
-		boolean breaksAlignment = true; //if result keys (matrix indexes) different 
-		boolean aligner = false; //if groups multiple inputs by key (e.g., group)
-		boolean definesMRJob = (et == ExecType.MR); //if requires its own MR job 
-		ExecLocation el = (et == ExecType.MR) ? ExecLocation.Map : ExecLocation.ControlProgram;
-		
-		lps.addCompatibility(JobType.GMR);
-		lps.setProperties( inputs, et, el, breaksAlignment, aligner, definesMRJob );
-	}
-
-	@Override
-	public String toString() {
-		return "Operation = MMTSJ";
-	}
-
-	/**
-	 * MR instruction generation.
-	 */
-	@Override
-	public String getInstructions(int input_index1, int output_index)
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-		sb.append( "tsmm" );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( this.prepOutputOperand(output_index));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( _type );
-		
-		return sb.toString();
-	}
-
-	/**
-	 * CP and Spark instruction generation.
-	 */
-	@Override
-	public String getInstructions(String input_index1, String output_index) throws LopsException
-	{	
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( "tsmm" );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( this.prepOutputOperand(output_index));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( _type );
-		
-		//append degree of parallelism for matrix multiplications
-		if( getExecType()==ExecType.CP ) {
-			sb.append( OPERAND_DELIMITOR );
-			sb.append( _numThreads );
-		}
-		
-		return sb.toString();
-	}
- 
- 
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MMZip.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MMZip.java b/src/main/java/com/ibm/bi/dml/lops/MMZip.java
deleted file mode 100644
index b8875fb..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MMZip.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-/**
- * Lop to perform zip matrix multiplication
- */
-public class MMZip extends Lop 
-{
-	private boolean _tRewrite = true;
-	
-	/**
-	 * 
-	 * @param input
-	 * @param op
-	 */
-	public MMZip(Lop input1, Lop input2, DataType dt, ValueType vt, boolean tRewrite, ExecType et) 
-	{
-		//handle inputs and outputs
-		super(Lop.Type.MMRJ, dt, vt);		
-		
-		_tRewrite = tRewrite;
-		
-		addInput(input1);
-		addInput(input2);
-		input1.addOutput(this);
-		input2.addOutput(this);
-		
-		//only applicable for spark
-		boolean breaksAlignment = false;
-		boolean aligner = false;
-		boolean definesMRJob = false;
-		lps.addCompatibility(JobType.INVALID);
-		lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
-	}
-
-	@Override
-	public String toString() {
-	
-		return "Operation = MMZip";
-	}
-
-	@Override
-	public String getInstructions(String input1, String input2, String output)
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-		sb.append( "zipmm" );
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(0).prepInputOperand(input1));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( getInputs().get(1).prepInputOperand(input2));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( prepOutputOperand(output));
-		sb.append( OPERAND_DELIMITOR );
-		sb.append( _tRewrite );
-		
-		return sb.toString();
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MapMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MapMult.java b/src/main/java/com/ibm/bi/dml/lops/MapMult.java
deleted file mode 100644
index e3d4d2e..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MapMult.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.AggBinaryOp.SparkAggType;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class MapMult extends Lop 
-{
-	
-	public static final String OPCODE = "mapmm";
-	
-	public enum CacheType {
-		RIGHT,
-		RIGHT_PART,
-		LEFT,
-		LEFT_PART;
-		
-		public boolean isRightCache(){
-			return (this == RIGHT || this == RIGHT_PART);
-		}
-	}
-	
-	private CacheType _cacheType = null;
-	private boolean _outputEmptyBlocks = true;
-	
-	//optional attribute for spark exec type
-	private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK;
-	
-	/**
-	 * Constructor to setup a partial Matrix-Vector Multiplication for MR
-	 * 
-	 * @param input
-	 * @param op
-	 * @return 
-	 * @throws LopsException
-	 */	
-	public MapMult(Lop input1, Lop input2, DataType dt, ValueType vt, boolean rightCache, boolean partitioned, boolean emptyBlocks ) 
-		throws LopsException 
-	{
-		super(Lop.Type.MapMult, dt, vt);		
-		this.addInput(input1);
-		this.addInput(input2);
-		input1.addOutput(this);
-		input2.addOutput(this);
-		
-		//setup mapmult parameters
-		if( rightCache )
-			_cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
-		else
-			_cacheType = partitioned ? CacheType.LEFT_PART : CacheType.LEFT;
-		_outputEmptyBlocks = emptyBlocks;
-		
-		//setup MR parameters 
-		boolean breaksAlignment = true;
-		boolean aligner = false;
-		boolean definesMRJob = false;
-		lps.addCompatibility(JobType.GMR);
-		lps.addCompatibility(JobType.DATAGEN);
-		lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
-	}
-
-	/**
-	 * Constructor to setup a partial Matrix-Vector Multiplication for Spark
-	 * 
-	 * @param input1
-	 * @param input2
-	 * @param dt
-	 * @param vt
-	 * @param rightCache
-	 * @param emptyBlocks
-	 * @param aggregate
-	 * @param et
-	 * @throws LopsException
-	 */
-	public MapMult(Lop input1, Lop input2, DataType dt, ValueType vt, boolean rightCache, boolean partitioned, boolean emptyBlocks, SparkAggType aggtype) 
-		throws LopsException 
-	{
-		super(Lop.Type.MapMult, dt, vt);		
-		this.addInput(input1);
-		this.addInput(input2);
-		input1.addOutput(this);
-		input2.addOutput(this);
-		
-		//setup mapmult parameters
-		if( rightCache )
-			_cacheType = partitioned ? CacheType.RIGHT_PART : CacheType.RIGHT;
-		else
-			_cacheType = partitioned ? CacheType.LEFT_PART : CacheType.LEFT;
-		_outputEmptyBlocks = emptyBlocks;
-		_aggtype = aggtype;
-		
-		//setup MR parameters 
-		boolean breaksAlignment = false;
-		boolean aligner = false;
-		boolean definesMRJob = false;
-		lps.addCompatibility(JobType.INVALID);
-		lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
-	}
-
-	public String toString() {
-		return "Operation = MapMM";
-	}
-	
-	@Override
-	public String getInstructions(int input_index1, int input_index2, int output_index)
-	{
-		//MR instruction generation
-		
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(OPCODE);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(1).prepInputOperand(input_index2));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( this.prepOutputOperand(output_index));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_cacheType);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_outputEmptyBlocks);
-		
-		return sb.toString();
-	}
-	
-	@Override
-	public String getInstructions(String input1, String input2, String output)
-	{
-		//Spark instruction generation
-		
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(OPCODE);
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append( getInputs().get(0).prepInputOperand(input1));
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		
-		sb.append( getInputs().get(1).prepInputOperand(input2));
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append( this.prepOutputOperand(output));
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(_cacheType);
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(_outputEmptyBlocks);
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(_aggtype.toString());
-		
-		return sb.toString();
-	}
-
-	@Override
-	public boolean usesDistributedCache() 
-	{
-		return true;
-	}
-	
-	@Override
-	public int[] distributedCacheInputIndex() 
-	{	
-		switch( _cacheType )
-		{
-			// first input is from distributed cache
-			case LEFT:
-			case LEFT_PART: 
-				return new int[]{1};
-			
-			// second input is from distributed cache
-			case RIGHT:
-			case RIGHT_PART: 
-				return new int[]{2};
-		}
-				
-		return new int[]{-1}; //error
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java b/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java
deleted file mode 100644
index 8badbc0..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/MapMultChain.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class MapMultChain extends Lop 
-{
-	
-	public static final String OPCODE = "mapmmchain";
-	public static final String OPCODE_CP = "mmchain";
-
-	public enum ChainType {
-		XtXv,  //(t(X) %*% (X %*% v))
-		XtwXv, //(t(X) %*% (w * (X %*% v)))
-		NONE,
-	}
-	
-	private ChainType _chainType = null;
-	private int _numThreads = 1;
-	
-	/**
-	 * Constructor to setup a map mult chain without weights
-	 * 
-	 * @param input
-	 * @param op
-	 * @return 
-	 * @throws LopsException
-	 */	
-	public MapMultChain(Lop input1, Lop input2, DataType dt, ValueType vt, ExecType et) 
-		throws LopsException 
-	{
-		super(Lop.Type.MapMultChain, dt, vt);		
-		addInput(input1); //X
-		addInput(input2); //v
-		input1.addOutput(this); 
-		input2.addOutput(this); 
-		
-		//setup mapmult parameters
-		_chainType = ChainType.XtXv;
-		setupLopProperties(et);
-	}
-	
-	/**
-	 * Constructor to setup a map mult chain with weights
-	 * 
-	 * @param input
-	 * @param op
-	 * @return 
-	 * @throws LopsException
-	 */	
-	public MapMultChain(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et) 
-		throws LopsException 
-	{
-		super(Lop.Type.MapMultChain, dt, vt);		
-		addInput(input1); //X
-		addInput(input2); //w
-		addInput(input3); //v
-		input1.addOutput(this);
-		input2.addOutput(this);
-		input3.addOutput(this);
-		
-		//setup mapmult parameters
-		_chainType = ChainType.XtwXv;
-		setupLopProperties(et);
-	}
-
-	/**
-	 * 
-	 * @param et
-	 */
-	private void setupLopProperties( ExecType et )
-	{
-		if( et == ExecType.MR )
-		{
-			//setup MR parameters 
-			boolean breaksAlignment = true;
-			boolean aligner = false;
-			boolean definesMRJob = false;
-			lps.addCompatibility(JobType.GMR);
-			lps.addCompatibility(JobType.DATAGEN);
-			lps.setProperties( inputs, ExecType.MR, ExecLocation.Map, breaksAlignment, aligner, definesMRJob );
-		}
-		else //Spark/CP
-		{
-			//setup Spark parameters 
-			boolean breaksAlignment = false;
-			boolean aligner = false;
-			boolean definesMRJob = false;
-			lps.addCompatibility(JobType.INVALID);
-			lps.setProperties( inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
-		}
-	}
-
-	public void setNumThreads(int k) {
-		_numThreads = k;
-	}
-	
-	public String toString() {
-		return "Operation = MapMMChain";
-	}
-	
-	@Override
-	public String getInstructions(int input_index1, int input_index2, int output_index)
-	{
-		//MR instruction XtXv
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(OPCODE);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(1).prepInputOperand(input_index2));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( this.prepOutputOperand(output_index));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_chainType);
-		
-		return sb.toString();
-	}
-	
-	@Override
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index)
-	{
-		//MR instruction XtwXv
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(OPCODE);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(1).prepInputOperand(input_index2));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(2).prepInputOperand(input_index3));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( this.prepOutputOperand(output_index));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_chainType);
-		
-		return sb.toString();
-	}
-
-	@Override
-	public String getInstructions(String input1, String input2, String output)
-	{
-		//Spark instruction XtXv
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		if( getExecType()==ExecType.CP )
-			sb.append(OPCODE_CP);
-		else
-			sb.append(OPCODE);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(0).prepInputOperand(input1));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(1).prepInputOperand(input2));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( this.prepOutputOperand(output));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_chainType);
-		
-		//append degree of parallelism for matrix multiplications
-		if( getExecType()==ExecType.CP ) {
-			sb.append( OPERAND_DELIMITOR );
-			sb.append( _numThreads );
-		}
-		
-		return sb.toString();
-	}
-	
-	@Override
-	public String getInstructions(String input1, String input2, String input3, String output)
-	{
-		//Spark instruction XtwXv
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		if( getExecType()==ExecType.CP )
-			sb.append(OPCODE_CP);
-		else
-			sb.append(OPCODE);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(0).prepInputOperand(input1));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(1).prepInputOperand(input2));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(2).prepInputOperand(input3));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( this.prepOutputOperand(output));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_chainType);
-		
-		//append degree of parallelism for matrix multiplications
-		if( getExecType()==ExecType.CP ) {
-			sb.append( OPERAND_DELIMITOR );
-			sb.append( _numThreads );
-		}
-		
-		return sb.toString();
-	}
-	
-	@Override
-	public boolean usesDistributedCache() 
-	{
-		return true;
-	}
-	
-	@Override
-	public int[] distributedCacheInputIndex() 
-	{
-		if( _chainType == ChainType.XtXv )
-			return new int[]{2};
-		else if( _chainType == ChainType.XtwXv )
-			return new int[]{2,3};
-		
-		//error
-		return new int[]{-1};
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java b/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java
deleted file mode 100644
index 3772f68..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/OutputParameters.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.hops.HopsException;
-
-/**
- * class to maintain output parameters for a lop.
- * 
- */
-
-public class OutputParameters 
-{
-	
-	public enum Format {
-		TEXT, BINARY, MM, CSV
-	};
-
-	private boolean _blocked = true;
-	private long _num_rows = -1;
-	private long _num_cols = -1;
-	private long _nnz = -1;	
-	private long _num_rows_in_block = -1;
-	private long _num_cols_in_block = -1;
-	private String _file_name = null;
-	private String _file_label = null;
-
-	Format matrix_format = Format.BINARY;
-	
-	public String getFile_name() {
-		return _file_name;
-	}
-
-	public void setFile_name(String fileName) {
-		_file_name = fileName;
-	}
-	
-	public String getLabel() {
-		return _file_label;
-	}
-
-	public void setLabel(String label) {
-		_file_label = label;
-	}
-
-	public void setDimensions(long rows, long cols, long rows_per_block, long cols_per_block, long nnz) throws HopsException {
-		_num_rows = rows;
-		_num_cols = cols;
-		_nnz = nnz;
-		_num_rows_in_block = rows_per_block;
-		_num_cols_in_block = cols_per_block;
-
-		if ( _num_rows_in_block == 0 && _num_cols_in_block == 0 ) {
-			_blocked = false;
-		}
-		else if (_num_rows_in_block == -1 && _num_cols_in_block == -1) {
-			_blocked = false;
- 		}
-		else if ( _num_rows_in_block > 0 && _num_cols_in_block > 0 ) {
-			_blocked = true;
-		}
-		else {
-			throw new HopsException("In OutputParameters Lop, Invalid values for blocking dimensions: [" + _num_rows_in_block + "," + _num_cols_in_block +"].");
-		}
-	}
-
-	public Format getFormat() {
-		return matrix_format;
-	}
-
-	public void setFormat(Format fmt) {
-		matrix_format = fmt;
-	}
-
-	public boolean isBlocked() {
-		return _blocked;
-	}
-
-	public void setBlocked(boolean blocked)
-	{
-		_blocked = blocked;
-	}
-	
-	public long getNumRows()
-	{
-		return _num_rows;
-	}
-	
-	public void setNumRows(long rows)
-	{
-		_num_rows = rows;
-	}
-	
-	public long getNumCols()
-	{
-		return _num_cols;
-	}
-	
-	public void setNumCols(long cols)
-	{
-		_num_cols = cols;
-	}
-	
-	public Long getNnz() {
-		return _nnz;
-	}
-	
-	public void setNnz(long nnz)
-	{
-		_nnz = nnz;
-	}
-
-	public long getRowsInBlock() {
-		return _num_rows_in_block;
-	}
-	
-	public void setRowsInBlock(long rows_in_block) {
-		_num_rows_in_block = rows_in_block;
-	}
-
-	public long getColsInBlock() {
-		return _num_cols_in_block;
-	}
-
-	public void setColsInBlock(long cols_in_block) {
-		_num_cols_in_block = cols_in_block;
-	}
-	
-	@Override
-	public String toString() {
-		StringBuilder sb = new StringBuilder();
-		sb.append("rows=" + getNumRows() + Lop.VALUETYPE_PREFIX);
-		sb.append("cols=" + getNumCols() + Lop.VALUETYPE_PREFIX);
-		sb.append("nnz=" + getNnz() + Lop.VALUETYPE_PREFIX);
-		sb.append("rowsInBlock=" + getRowsInBlock() + Lop.VALUETYPE_PREFIX);
-		sb.append("colsInBlock=" + getColsInBlock() + Lop.VALUETYPE_PREFIX);
-		sb.append("isBlockedRepresentation=" + isBlocked() + Lop.VALUETYPE_PREFIX);
-		sb.append("format=" + getFormat() + Lop.VALUETYPE_PREFIX);
-		sb.append("label=" + getLabel() + Lop.VALUETYPE_PREFIX);
-		sb.append("filename=" + getFile_name());
-		return sb.toString();
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PMMJ.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PMMJ.java b/src/main/java/com/ibm/bi/dml/lops/PMMJ.java
deleted file mode 100644
index c540256..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PMMJ.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-/**
- *
- */
-public class PMMJ extends Lop 
-{
-	
-	public static final String OPCODE = "pmm";
-	
-	public enum CacheType {
-		LEFT,
-		LEFT_PART;
-	}
-	
-	private CacheType _cacheType = null;
-	private boolean _outputEmptyBlocks = true;
-	private int _numThreads = 1;
-	
-	/**
-	 * Constructor to setup a Permutation Matrix Multiplication
-	 * 
-	 * @param input
-	 * @param op
-	 * @return 
-	 * @throws LopsException
-	 */	
-	public PMMJ(Lop pminput, Lop rightinput, Lop nrow, DataType dt, ValueType vt, boolean partitioned, boolean emptyBlocks, ExecType et) 
-		throws LopsException 
-	{
-		super(Lop.Type.PMMJ, dt, vt);		
-		addInput(pminput);
-		addInput(rightinput);
-		addInput(nrow);
-		pminput.addOutput(this);
-		rightinput.addOutput(this);
-		nrow.addOutput(this);
-		
-		//setup mapmult parameters
-		_cacheType = partitioned ? CacheType.LEFT_PART : CacheType.LEFT;
-		_outputEmptyBlocks = emptyBlocks;
-		
-		boolean breaksAlignment = true;
-		boolean aligner = false;
-		boolean definesMRJob = false;
-		ExecLocation el = (et == ExecType.MR) ? ExecLocation.Map : ExecLocation.ControlProgram;
-		lps.addCompatibility(JobType.GMR);
-		lps.addCompatibility(JobType.DATAGEN);
-		lps.setProperties( inputs, et, el, breaksAlignment, aligner, definesMRJob );
-	}
-
-	@Override
-	public String toString() {
-		return "Operation = PMMJ";
-	}
-	
-	@Override
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index)
-	{
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(OPCODE);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(1).prepInputOperand(input_index2));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(2).prepScalarLabel() );
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( this.prepOutputOperand(output_index));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_cacheType);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append(_outputEmptyBlocks);
-		
-		return sb.toString();
-	}
-	
-	@Override
-	public String getInstructions(String input_index1, String input_index2, String input_index3, String output_index) 
-		throws LopsException
-	{	
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(OPCODE);
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(0).prepInputOperand(input_index1));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(1).prepInputOperand(input_index2));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( getInputs().get(2).prepInputOperand(input_index3));
-		
-		sb.append(Lop.OPERAND_DELIMITOR);
-		sb.append( this.prepOutputOperand(output_index));
-		
-		if( getExecType() == ExecType.SPARK ) 
-		{
-			sb.append(Lop.OPERAND_DELIMITOR);
-			sb.append(_cacheType);
-		}
-		else if( getExecType()==ExecType.CP ) {
-			//append degree of parallelism
-			sb.append( OPERAND_DELIMITOR );
-			sb.append( _numThreads );
-		}
-		
-		return sb.toString();
-	}
-
-	@Override
-	public boolean usesDistributedCache() 
-	{
-		return true;
-	}
-	
-	@Override
-	public int[] distributedCacheInputIndex() 
-	{	
-		//always left cached selection vector
-		return new int[]{1};
-	}
-	
-	public void setNumThreads(int k) {
-		_numThreads = k;
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/PMapMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/PMapMult.java b/src/main/java/com/ibm/bi/dml/lops/PMapMult.java
deleted file mode 100644
index cb12999..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/PMapMult.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-
-
-public class PMapMult extends Lop 
-{	
-	public static final String OPCODE = "pmapmm";
-
-	/**
-	 * 
-	 * @param input1
-	 * @param input2
-	 * @param dt
-	 * @param vt
-	 * @param rightCache
-	 * @param emptyBlocks
-	 * @param aggregate
-	 * @param et
-	 * @throws LopsException
-	 */
-	public PMapMult(Lop input1, Lop input2, DataType dt, ValueType vt) 
-		throws LopsException 
-	{
-		super(Lop.Type.MapMult, dt, vt);		
-		this.addInput(input1);
-		this.addInput(input2);
-		input1.addOutput(this);
-		input2.addOutput(this);
-		
-		//setup MR parameters 
-		boolean breaksAlignment = false;
-		boolean aligner = false;
-		boolean definesMRJob = false;
-		lps.addCompatibility(JobType.INVALID);
-		lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob );
-	}
-
-	public String toString() {
-		return "Operation = PMapMM";
-	}
-	
-	@Override
-	public String getInstructions(String input1, String input2, String output)
-	{
-		//Spark instruction generation		
-		StringBuilder sb = new StringBuilder();
-		
-		sb.append(getExecType());
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append(OPCODE);
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append( getInputs().get(0).prepInputOperand(input1));
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append( getInputs().get(1).prepInputOperand(input2));
-		sb.append(Lop.OPERAND_DELIMITOR);
-		
-		sb.append( prepOutputOperand(output));
-		
-		return sb.toString();
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java b/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java
deleted file mode 100644
index c67a92b..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/ParameterizedBuiltin.java
+++ /dev/null
@@ -1,514 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- */
-
-package com.ibm.bi.dml.lops;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecLocation;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ParameterizedBuiltinFunctionExpression;
-
-
-/**
- * Defines a LOP for functions.
- * 
- */
-public class ParameterizedBuiltin extends Lop 
-{
-	
-	public enum OperationTypes { 
-		INVALID, CDF, INVCDF, RMEMPTY, REPLACE, REXPAND, 
-		PNORM, QNORM, PT, QT, PF, QF, PCHISQ, QCHISQ, PEXP, QEXP,
-		TRANSFORM
-	};
-	
-	private OperationTypes _operation;
-	private HashMap<String, Lop> _inputParams;
-	private boolean _bRmEmptyBC;
-
-	/**
-	 * Creates a new builtin function LOP.
-	 * 
-	 * @param target
-	 *            target identifier
-	 * @param params
-	 *            parameter list
-	 * @param inputParameters
-	 *            list of input LOPs
-	 * @param function
-	 *            builtin function
-	 * @param numRows
-	 *            number of resulting rows
-	 * @param numCols
-	 *            number of resulting columns
-	 */
-	public ParameterizedBuiltin(HashMap<String, Lop> paramLops, OperationTypes op, DataType dt, ValueType vt) 
-	{
-		super(Lop.Type.ParameterizedBuiltin, dt, vt);
-		_operation = op;
-		
-		for (Lop lop : paramLops.values()) {
-			this.addInput(lop);
-			lop.addOutput(this);
-		}
-		
-		_inputParams = paramLops;
-		
-		/*
-		 * This lop is executed in control program. 
-		 */
-		boolean breaksAlignment = false;
-		boolean aligner = false;
-		boolean definesMRJob = false;
-		lps.addCompatibility(JobType.INVALID);
-		lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob);
-	}
-	
-	public ParameterizedBuiltin(HashMap<String, Lop> paramLops, OperationTypes op, DataType dt, ValueType vt, ExecType et) 
-		throws HopsException 
-	{
-		super(Lop.Type.ParameterizedBuiltin, dt, vt);
-		_operation = op;
-		
-		for (Lop lop : paramLops.values()) {
-			this.addInput(lop);
-			lop.addOutput(this);
-		}
-		
-		_inputParams = paramLops;
-		
-		boolean breaksAlignment = false;
-		boolean aligner = false;
-		boolean definesMRJob = false;
-		ExecLocation eloc = null;
-		
-		if( _operation == OperationTypes.REPLACE && et==ExecType.MR )
-		{
-			eloc = ExecLocation.MapOrReduce;
-			lps.addCompatibility(JobType.GMR);
-			lps.addCompatibility(JobType.DATAGEN);
-			lps.addCompatibility(JobType.REBLOCK);
-		}
-		else if( _operation == OperationTypes.RMEMPTY && et==ExecType.MR )
-		{
-			eloc = ExecLocation.Reduce;
-			lps.addCompatibility(JobType.GMR);
-			lps.addCompatibility(JobType.DATAGEN);
-			lps.addCompatibility(JobType.REBLOCK);
-			breaksAlignment=true;
-		}
-		else if( _operation == OperationTypes.REXPAND && et==ExecType.MR )
-		{
-			eloc = ExecLocation.MapOrReduce;
-			lps.addCompatibility(JobType.GMR);
-			lps.addCompatibility(JobType.DATAGEN);
-			lps.addCompatibility(JobType.REBLOCK);
-			breaksAlignment=true;
-		}
-		else if ( _operation == OperationTypes.TRANSFORM && et == ExecType.MR ) {
-			definesMRJob = true;
-			eloc = ExecLocation.MapAndReduce;
-			lps.addCompatibility(JobType.TRANSFORM);
-		}
-		else //executed in CP / CP_FILE / SPARK
-		{
-			eloc = ExecLocation.ControlProgram;
-			lps.addCompatibility(JobType.INVALID);
-		}
-		lps.setProperties(inputs, et, eloc, breaksAlignment, aligner, definesMRJob);
-	}
-
-	public ParameterizedBuiltin(HashMap<String, Lop> paramLops, OperationTypes op, DataType dt, ValueType vt, ExecType et, boolean bRmEmptyBC) 
-			throws HopsException 
-	{
-		this(paramLops, op, dt, vt, et);
-		_bRmEmptyBC = bRmEmptyBC;
-	}
-	
-	public OperationTypes getOp() { 
-		return _operation; 
-	}
-	
-	public int getInputIndex(String name) { 
-		Lop n = _inputParams.get(name);
-		for(int i=0; i<getInputs().size(); i++) 
-			if(getInputs().get(i) == n)
-				return i;
-		return -1;
-	}
-	
-	public Lop getNamedInput(String name) {
-		return _inputParams.get(name);
-	}
-	
-	@Override
-	public String getInstructions(String output) 
-		throws LopsException 
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-
-		switch(_operation) 
-		{
-			case CDF:
-			case INVCDF:
-				sb.append( (_operation == OperationTypes.CDF ? "cdf" : "invcdf") );
-				sb.append( OPERAND_DELIMITOR );
-				
-				for ( String s : _inputParams.keySet() ) 
-				{	
-					sb.append( s );
-					sb.append( NAME_VALUE_SEPARATOR );
-					
-					// get the value/label of the scalar input associated with name "s"
-					Lop iLop = _inputParams.get(s);
-					sb.append( iLop.prepScalarLabel() );
-					sb.append( OPERAND_DELIMITOR );
-				}
-				break;
-				
-			case RMEMPTY:
-				sb.append("rmempty");
-				sb.append(OPERAND_DELIMITOR);
-				
-				for ( String s : _inputParams.keySet() ) {
-					
-					sb.append(s);
-					sb.append(NAME_VALUE_SEPARATOR);
-					
-					// get the value/label of the scalar input associated with name "s"
-					// (offset and maxdim only apply to exec type spark)
-					Lop iLop = _inputParams.get(s);
-					if( s.equals( "target") || s.equals( "select") || getExecType()==ExecType.SPARK )
-						sb.append( iLop.getOutputParameters().getLabel());
-					else
-						sb.append( iLop.prepScalarLabel() );
-					
-					sb.append(OPERAND_DELIMITOR);
-				}
-				
-				break;
-			
-			case REPLACE:
-				sb.append( "replace" );
-				sb.append( OPERAND_DELIMITOR );
-				
-				for ( String s : _inputParams.keySet() ) 
-				{	
-					sb.append( s );
-					sb.append( NAME_VALUE_SEPARATOR );
-					
-					// get the value/label of the scalar input associated with name "s"
-					Lop iLop = _inputParams.get(s);
-					if( s.equals("target") )
-						sb.append(iLop.getOutputParameters().getLabel());
-					else
-						sb.append( iLop.prepScalarLabel() );
-					sb.append( OPERAND_DELIMITOR );
-				}
-				break;
-			
-			case REXPAND:
-				sb.append("rexpand");
-				sb.append(OPERAND_DELIMITOR);
-				
-				for ( String s : _inputParams.keySet() ) {
-					
-					sb.append(s);
-					sb.append(NAME_VALUE_SEPARATOR);
-					
-					// get the value/label of the scalar input associated with name "s"
-					// (offset and maxdim only apply to exec type spark)
-					Lop iLop = _inputParams.get(s);
-					if( s.equals( "target") || getExecType()==ExecType.SPARK )
-						sb.append( iLop.getOutputParameters().getLabel());
-					else
-						sb.append( iLop.prepScalarLabel() );
-					
-					sb.append(OPERAND_DELIMITOR);
-				}
-				
-				break;
-				
-			case TRANSFORM:
-			{
-				sb.append("transform");
-				sb.append(OPERAND_DELIMITOR);
-				
-				for ( String s : _inputParams.keySet() ) {
-					sb.append(s);
-					sb.append(NAME_VALUE_SEPARATOR);
-					
-					Lop iLop = _inputParams.get(s);
-					if( iLop.getDataType() != DataType.SCALAR )
-						sb.append( iLop.getOutputParameters().getLabel());
-					else
-						sb.append( iLop.prepScalarLabel() );
-					
-					sb.append(OPERAND_DELIMITOR);
-				}
-				break;
-			}
-				
-			default:
-				throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
-		}
-		
-		if (_operation == OperationTypes.RMEMPTY) {			
-			sb.append("bRmEmptyBC");
-			sb.append(NAME_VALUE_SEPARATOR);
-			sb.append( _bRmEmptyBC );
-			sb.append(OPERAND_DELIMITOR);
-		}
-
-		sb.append(this.prepOutputOperand(output));
-		
-		return sb.toString();
-	}
-
-	@Override 
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int output_index) 
-		throws LopsException
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-
-		switch(_operation) 
-		{
-			case REPLACE:
-			{
-				sb.append( "replace" );
-				sb.append( OPERAND_DELIMITOR );
-		
-				Lop iLop = _inputParams.get("target");
-				int pos = getInputs().indexOf(iLop);
-				int index = (pos==0)? input_index1 : (pos==1)? input_index2 : input_index3;
-				//input_index
-				sb.append(prepInputOperand(index));
-				sb.append( OPERAND_DELIMITOR );
-				
-				Lop iLop2 = _inputParams.get("pattern");
-				sb.append(iLop2.prepScalarLabel());
-				sb.append( OPERAND_DELIMITOR );
-				
-				Lop iLop3 = _inputParams.get("replacement");
-				sb.append(iLop3.prepScalarLabel());
-				sb.append( OPERAND_DELIMITOR );
-				
-				break;
-			}	
-				
-			default:
-				throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
-		}
-		
-		sb.append( prepOutputOperand(output_index));
-		
-		return sb.toString();
-	}
-
-	@Override 
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int output_index) 
-		throws LopsException
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-
-		switch(_operation) 
-		{
-			case RMEMPTY:
-			{
-				sb.append("rmempty");
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop1 = _inputParams.get("target");
-				int pos1 = getInputs().indexOf(iLop1);
-				int index1 = (pos1==0)? input_index1 : (pos1==1)? input_index2 : (pos1==2)? input_index3 : input_index4;
-				sb.append(prepInputOperand(index1));
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop2 = _inputParams.get("offset");
-				int pos2 = getInputs().indexOf(iLop2);
-				int index2 = (pos2==0)? input_index1 : (pos2==1)? input_index2 : (pos1==2)? input_index3 : input_index4;
-				sb.append(prepInputOperand(index2));
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop3 = _inputParams.get("maxdim");
-				sb.append( iLop3.prepScalarLabel() );
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop4 = _inputParams.get("margin");
-				sb.append( iLop4.prepScalarLabel() );
-				
-				sb.append( OPERAND_DELIMITOR );
-				
-				break;
-			}
-				
-			default:
-				throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
-		}
-		
-		sb.append( prepOutputOperand(output_index));
-		
-		return sb.toString();
-	}
-	
-	@Override 
-	public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index) 
-		throws LopsException
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-
-		switch(_operation) 
-		{
-			case REXPAND:
-			{
-				sb.append("rexpand");
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop1 = _inputParams.get("target");
-				int pos1 = getInputs().indexOf(iLop1);
-				int index1 = (pos1==0)? input_index1 : (pos1==1)? input_index2 : (pos1==2)? input_index3 : (pos1==3)? input_index4 : input_index5;
-				sb.append(prepInputOperand(index1));
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop2 = _inputParams.get("max");
-				sb.append( iLop2.prepScalarLabel() );
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop3 = _inputParams.get("dir");
-				sb.append( iLop3.prepScalarLabel() );
-				
-				sb.append(OPERAND_DELIMITOR);
-				
-				Lop iLop4 = _inputParams.get("cast");
-				sb.append( iLop4.prepScalarLabel() );
-				
-				sb.append( OPERAND_DELIMITOR );
-				
-				Lop iLop5 = _inputParams.get("ignore");
-				sb.append( iLop5.prepScalarLabel() );
-				
-				sb.append( OPERAND_DELIMITOR );
-				
-				break;
-			}
-				
-			default:
-				throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
-		}
-		
-		sb.append( prepOutputOperand(output_index));
-		
-		return sb.toString();
-	}
-	
-	@Override 
-	public String getInstructions(int output_index) 
-		throws LopsException
-	{
-		StringBuilder sb = new StringBuilder();
-		sb.append( getExecType() );
-		sb.append( Lop.OPERAND_DELIMITOR );
-
-		if(_operation== OperationTypes.TRANSFORM) 
-		{
-			// int inputIndex = getInputIndex("target");
-			
-			sb.append( "transform" );
-			sb.append( OPERAND_DELIMITOR );
-			Lop iLop = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_DATA);
-			sb.append(iLop.prepInputOperand(getInputIndex("target")));
-			sb.append( OPERAND_DELIMITOR );
-			
-			Lop iLop2 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_TXMTD);
-			sb.append(iLop2.prepScalarLabel());
-			sb.append( OPERAND_DELIMITOR );
-			
-			// either applyTransformPath or transformSpec should be specified
-			boolean isApply = false;
-			Lop iLop3 = null;
-			if ( _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_APPLYMTD) != null ) {
-				// apply transform
-				isApply = true;
-				iLop3 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_APPLYMTD);
-			}
-			else {
-				iLop3 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_TXSPEC);
-			}
-			sb.append(iLop3.prepScalarLabel());
-			sb.append( OPERAND_DELIMITOR );
-			
-			sb.append(isApply);
-			sb.append( OPERAND_DELIMITOR );
-			
-			Lop iLop4 = _inputParams.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_OUTNAMES);
-			if( iLop4 != null ) 
-			{
-				sb.append(iLop4.prepScalarLabel());
-				sb.append( OPERAND_DELIMITOR );
-			}
-			
-			sb.append( prepOutputOperand(output_index));
-		}
-			//return getTransformInstructions(""+getInputIndex("target"), ""+output_index);
-		else
-			throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
-	
-		return sb.toString();
-	}
-	
-
-	@Override
-	public String toString() {
-		StringBuilder sb = new StringBuilder();
-		sb.append(_operation.toString());
-
-		if( !getInputs().isEmpty() )
-			sb.append("(");
-		for (Lop cur : getInputs()) {
-			sb.append(cur.toString());
-		}
-		if( !getInputs().isEmpty() )
-			sb.append(") ");
-
-		sb.append(" ; num_rows=" + this.getOutputParameters().getNumRows());
-		sb.append(" ; num_cols=" + this.getOutputParameters().getNumCols());
-		sb.append(" ; format=" + this.getOutputParameters().getFormat());
-		sb.append(" ; blocked=" + this.getOutputParameters().isBlocked());
-		return sb.toString();
-	}
-
-}