You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:46:14 UTC
[43/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java b/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
deleted file mode 100644
index 90d5ec0..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
+++ /dev/null
@@ -1,532 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.rewrite.HopRewriteUtils;
-import com.ibm.bi.dml.hops.Hop.MultiThreadedHop;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.DataGen;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.DataExpression;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.Statement;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-
-/**
- *
- *
- */
-public class DataGenOp extends Hop implements MultiThreadedHop
-{
-
- public static final long UNSPECIFIED_SEED = -1;
-
- // defines the specific data generation method
- private DataGenMethod _op;
- private int _maxNumThreads = -1; //-1 for unlimited
-
- /**
- * List of "named" input parameters. They are maintained as a hashmap:
- * parameter names (String) are mapped as indices (Integer) into getInput()
- * arraylist.
- *
- * i.e., getInput().get(_paramIndexMap.get(parameterName)) refers to the Hop
- * that is associated with parameterName.
- */
- private HashMap<String, Integer> _paramIndexMap = new HashMap<String, Integer>();
-
-
- /** target identifier which will hold the random object */
- private DataIdentifier _id;
-
- //Rand-specific attributes
-
- /** sparsity of the random object, this is used for mem estimate */
- private double _sparsity = -1;
- /** base directory for temp file (e.g., input seeds)*/
- private String _baseDir;
-
- //seq-specific attributes (used for recompile/recompile)
- private double _incr = Double.MAX_VALUE;
-
-
- private DataGenOp() {
- //default constructor for clone
- }
-
- /**
- * <p>Creates a new Rand HOP.</p>
- *
- * @param id the target identifier
- * @param inputParameters HashMap of the input parameters for Rand Hop
- */
- public DataGenOp(DataGenMethod mthd, DataIdentifier id, HashMap<String, Hop> inputParameters)
- {
- super(id.getName(), DataType.MATRIX, ValueType.DOUBLE);
-
- _id = id;
- _op = mthd;
-
- int index = 0;
- for( Entry<String, Hop> e: inputParameters.entrySet() ) {
- String s = e.getKey();
- Hop input = e.getValue();
- getInput().add(input);
- input.getParent().add(this);
-
- _paramIndexMap.put(s, index);
- index++;
- }
- if ( mthd == DataGenMethod.RAND )
- _sparsity = Double.valueOf(((LiteralOp)inputParameters.get(DataExpression.RAND_SPARSITY)).getName());
-
- //generate base dir
- String scratch = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
- _baseDir = scratch + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- Lop.FILE_SEPARATOR + ProgramConverter.CP_ROOT_THREAD_ID + Lop.FILE_SEPARATOR;
-
- //compute unknown dims and nnz
- refreshSizeInformation();
- }
-
- @Override
- public String getOpString() {
- return "dg(" + _op.toString().toLowerCase() +")";
- }
-
- public DataGenMethod getOp() {
- return _op;
- }
-
- @Override
- public void setMaxNumThreads( int k ) {
- _maxNumThreads = k;
- }
-
- @Override
- public int getMaxNumThreads() {
- return _maxNumThreads;
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- ExecType et = optFindExecType();
-
- HashMap<String, Lop> inputLops = new HashMap<String, Lop>();
- for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
- if( cur.getKey().equals(DataExpression.RAND_ROWS) && _dim1>0 )
- inputLops.put(cur.getKey(), new LiteralOp(_dim1).constructLops());
- else if( cur.getKey().equals(DataExpression.RAND_COLS) && _dim2>0 )
- inputLops.put(cur.getKey(), new LiteralOp(_dim2).constructLops());
- else
- inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
- }
-
- DataGen rnd = new DataGen(_op, _id, inputLops,_baseDir,
- getDataType(), getValueType(), et);
-
- int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
- rnd.setNumThreads(k);
-
- rnd.getOutputParameters().setDimensions(
- getDim1(), getDim2(),
- //robust handling for blocksize (important for -exec singlenode; otherwise incorrect results)
- (getRowsInBlock()>0)?getRowsInBlock():DMLTranslator.DMLBlockSize,
- (getColsInBlock()>0)?getColsInBlock():DMLTranslator.DMLBlockSize,
- //actual rand nnz might differ (in cp/mr they are corrected after execution)
- (_op==DataGenMethod.RAND && et==ExecType.SPARK && getNnz()!=0) ? -1 : getNnz() );
-
- setLineNumbers(rnd);
- setLops(rnd);
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- @Override
- public void printMe() throws HopsException
- {
- if (LOG.isDebugEnabled()){
- if(getVisited() != VisitStatus.DONE)
- {
- super.printMe();
- }
-
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return true;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- if ( _op == DataGenMethod.RAND ) {
- if( hasConstantValue(0.0) ) { //if empty block
- ret = OptimizerUtils.estimateSizeEmptyBlock(dim1, dim2);
- }
- else {
- //sparsity-aware estimation (dependent on sparse generation approach); for pure dense generation
- //we would need to disable sparsity-awareness and estimate via sparsity=1.0
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity);
- }
- }
- else {
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0);
- }
-
- return ret;
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- if ( _op == DataGenMethod.RAND && dimsKnown() ) {
- long numBlocks = (long) (Math.ceil((double)dim1/DMLTranslator.DMLBlockSize) * Math.ceil((double)dim2/DMLTranslator.DMLBlockSize));
- return 32 + numBlocks*8.0; // 32 bytes of overhead for an array of long & numBlocks long values.
- }
- else
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- //infer rows and
- if( (_op == DataGenMethod.RAND || _op == DataGenMethod.SINIT ) &&
- OptimizerUtils.ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION )
- {
- long dim1 = computeDimParameterInformation(getInput().get(_paramIndexMap.get(DataExpression.RAND_ROWS)), memo);
- long dim2 = computeDimParameterInformation(getInput().get(_paramIndexMap.get(DataExpression.RAND_COLS)), memo);
- long nnz = (long)(_sparsity * dim1 * dim2);
- if( dim1>0 && dim2>0 )
- return new long[]{ dim1, dim2, nnz };
- }
- else if ( _op == DataGenMethod.SEQ )
- {
- Hop from = getInput().get(_paramIndexMap.get(Statement.SEQ_FROM));
- Hop to = getInput().get(_paramIndexMap.get(Statement.SEQ_TO));
- Hop incr = getInput().get(_paramIndexMap.get(Statement.SEQ_INCR));
- //in order to ensure correctness we also need to know from and incr
- //here, we check for the common case of seq(1,x), i.e. from=1, incr=1
- if( from instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)from)==1
- && incr instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)incr)==1 )
- {
- long toVal = computeDimParameterInformation(to, memo);
- if( toVal > 0 )
- return new long[]{ toVal, 1, -1 };
- }
- //here, we check for the common case of seq(1,x,?), i.e., from=1, to=x, b(seqincr) operator
- if( from instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)from)==1
- && incr instanceof BinaryOp && ((BinaryOp)incr).getOp() == Hop.OpOp2.SEQINCR ) //implicit 1
- {
- long toVal = computeDimParameterInformation(to, memo);
- if( toVal > 0 )
- return new long[]{ toVal, 1, -1 };
- }
- //here, we check for the common case of seq(x,1,-1), i.e. from=x, to=1 incr=-1
- if( to instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)to)==1
- && incr instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)incr)==-1 )
- {
- long fromVal = computeDimParameterInformation(from, memo);
- if( fromVal > 0 )
- return new long[]{ fromVal, 1, -1 };
- }
- }
-
- return null;
- }
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
-
- checkAndSetForcedPlatform();
-
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- if( _etypeForced != null )
- _etype = _etypeForced;
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
- _etype = findExecTypeByMemEstimate();
- }
- else if (this.areDimsBelowThreshold() || this.isVector())
- _etype = ExecType.CP;
- else
- _etype = REMOTE;
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
- setRequiresRecompile();
-
- //always force string initialization into CP (not supported in MR)
- //similarly, sample is currently not supported in MR either
- if( _op == DataGenMethod.SINIT )
- _etype = ExecType.CP;
-
- //workaround until sample supported in MR
- if( _op == DataGenMethod.SAMPLE && _etype == ExecType.MR )
- _etype = ExecType.CP;
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- Hop input1 = null;
- Hop input2 = null;
- Hop input3 = null;
-
- if ( _op == DataGenMethod.RAND || _op == DataGenMethod.SINIT )
- {
- input1 = getInput().get(_paramIndexMap.get(DataExpression.RAND_ROWS)); //rows
- input2 = getInput().get(_paramIndexMap.get(DataExpression.RAND_COLS)); //cols
-
- //refresh rows information
- refreshRowsParameterInformation(input1);
-
- //refresh cols information
- refreshColsParameterInformation(input2);
- }
- else if (_op == DataGenMethod.SEQ )
- {
- input1 = getInput().get(_paramIndexMap.get(Statement.SEQ_FROM));
- input2 = getInput().get(_paramIndexMap.get(Statement.SEQ_TO));
- input3 = getInput().get(_paramIndexMap.get(Statement.SEQ_INCR));
-
- double from = computeBoundsInformation(input1);
- boolean fromKnown = (from != Double.MAX_VALUE);
-
- double to = computeBoundsInformation(input2);
- boolean toKnown = (to != Double.MAX_VALUE);
-
- double incr = computeBoundsInformation(input3);
- boolean incrKnown = (incr != Double.MAX_VALUE);
-
- if( !incrKnown && input3 instanceof BinaryOp //special case for incr
- && ((BinaryOp)input3).getOp() == Hop.OpOp2.SEQINCR && fromKnown && toKnown)
- {
- incr = ( from >= to ) ? -1 : 1;
- incrKnown = true;
- }
-
- if ( fromKnown && toKnown && incrKnown ) {
- setDim1(1 + (long)Math.floor(((double)(to-from))/incr));
- setDim2(1);
- _incr = incr;
- }
- }
-
- //refresh nnz information (for seq, sparsity is always -1)
- if( _op == DataGenMethod.RAND && hasConstantValue(0.0) )
- _nnz = 0;
- else if ( dimsKnown() && _sparsity>=0 ) //general case
- _nnz = (long) (_sparsity * _dim1 * _dim2);
- }
-
-
- public HashMap<String, Integer> getParamIndexMap()
- {
- return _paramIndexMap;
- }
-
- public int getParamIndex(String key)
- {
- return _paramIndexMap.get(key);
- }
-
- /**
- *
- * @return
- */
- public boolean hasConstantValue()
- {
- //string initialization does not exhibit constant values
- if( _op == DataGenMethod.SINIT )
- return false;
-
- Hop min = getInput().get(_paramIndexMap.get(DataExpression.RAND_MIN)); //min
- Hop max = getInput().get(_paramIndexMap.get(DataExpression.RAND_MAX)); //max
- Hop sparsity = getInput().get(_paramIndexMap.get(DataExpression.RAND_SPARSITY)); //sparsity
-
- //literal value comparison
- if( min instanceof LiteralOp && max instanceof LiteralOp && sparsity instanceof LiteralOp){
- try{
- double minVal = HopRewriteUtils.getDoubleValue((LiteralOp)min);
- double maxVal = HopRewriteUtils.getDoubleValue((LiteralOp)max);
- double sp = HopRewriteUtils.getDoubleValue((LiteralOp)sparsity);
- return (sp==1.0 && minVal == maxVal);
- }
- catch(Exception ex)
- {
- return false;
- }
- }
- //reference comparison (based on common subexpression elimination)
- else if ( min == max && sparsity instanceof LiteralOp ) {
- try {
- double sp = HopRewriteUtils.getDoubleValue((LiteralOp)sparsity);
- return (sp==1.0);
- }
- catch(Exception ex)
- {
- return false;
- }
- }
-
- return false;
- }
-
- /**
- *
- * @param val
- * @return
- */
- public boolean hasConstantValue(double val)
- {
- //string initialization does not exhibit constant values
- if( _op == DataGenMethod.SINIT )
- return false;
-
- boolean ret = false;
-
- Hop min = getInput().get(_paramIndexMap.get(DataExpression.RAND_MIN)); //min
- Hop max = getInput().get(_paramIndexMap.get(DataExpression.RAND_MAX)); //max
-
- //literal value comparison
- if( min instanceof LiteralOp && max instanceof LiteralOp){
- double minVal = HopRewriteUtils.getDoubleValueSafe((LiteralOp)min);
- double maxVal = HopRewriteUtils.getDoubleValueSafe((LiteralOp)max);
- ret = (minVal == val && maxVal == val);
- }
-
- //sparsity awareness if requires
- if( ret && val != 0 ) {
- Hop sparsity = getInput().get(_paramIndexMap.get(DataExpression.RAND_SPARSITY)); //sparsity
- ret &= (sparsity == null || sparsity instanceof LiteralOp
- && HopRewriteUtils.getDoubleValueSafe((LiteralOp)sparsity) == 1);
- }
-
- return ret;
- }
-
- public void setIncrementValue(double incr)
- {
- _incr = incr;
- }
-
- public double getIncrementValue()
- {
- return _incr;
- }
-
- public static long generateRandomSeed()
- {
- return System.nanoTime();
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Object clone() throws CloneNotSupportedException
- {
- DataGenOp ret = new DataGenOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._op = _op;
- ret._id = _id;
- ret._sparsity = _sparsity;
- ret._baseDir = _baseDir;
- ret._paramIndexMap = (HashMap<String, Integer>) _paramIndexMap.clone();
- ret._maxNumThreads = _maxNumThreads;
- //note: no deep cp of params since read-only
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- if( !(that instanceof DataGenOp) )
- return false;
-
- DataGenOp that2 = (DataGenOp)that;
- boolean ret = ( _op == that2._op
- && _sparsity == that2._sparsity
- && _baseDir.equals(that2._baseDir)
- && _paramIndexMap!=null && that2._paramIndexMap!=null
- && _maxNumThreads == that2._maxNumThreads );
-
- if( ret )
- {
- for( Entry<String,Integer> e : _paramIndexMap.entrySet() )
- {
- String key1 = e.getKey();
- int pos1 = e.getValue();
- int pos2 = that2._paramIndexMap.get(key1);
- ret &= ( that2.getInput().get(pos2)!=null
- && getInput().get(pos1) == that2.getInput().get(pos2) );
- }
-
- //special case for rand seed (no CSE if unspecified seed because runtime generated)
- //note: if min and max is constant, we can safely merge those hops
- if( _op == DataGenMethod.RAND || _op == DataGenMethod.SINIT ){
- Hop seed = getInput().get(_paramIndexMap.get(DataExpression.RAND_SEED));
- Hop min = getInput().get(_paramIndexMap.get(DataExpression.RAND_MIN));
- Hop max = getInput().get(_paramIndexMap.get(DataExpression.RAND_MAX));
- if( seed.getName().equals(String.valueOf(DataGenOp.UNSPECIFIED_SEED)) && min != max )
- ret = false;
- }
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/DataOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/DataOp.java b/src/main/java/com/ibm/bi/dml/hops/DataOp.java
deleted file mode 100644
index 3975ed9..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/DataOp.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-
-public class DataOp extends Hop
-{
-
- private DataOpTypes _dataop;
- private String _fileName = null;
-
- //read dataop properties
- private FileFormatTypes _inFormat = FileFormatTypes.TEXT;
- private long _inRowsInBlock = -1;
- private long _inColsInBlock = -1;
-
- private boolean _recompileRead = true;
-
- /**
- * List of "named" input parameters. They are maintained as a hashmap:
- * parameter names (String) are mapped as indices (Integer) into getInput()
- * arraylist.
- *
- * i.e., getInput().get(_paramIndexMap.get(parameterName)) refers to the Hop
- * that is associated with parameterName.
- */
- private HashMap<String, Integer> _paramIndexMap = new HashMap<String, Integer>();
-
- private DataOp() {
- //default constructor for clone
- }
-
- /**
- * READ operation for Matrix w/ dim1, dim2.
- * This constructor does not support any expression in parameters
- */
- public DataOp(String l, DataType dt, ValueType vt, DataOpTypes dop,
- String fname, long dim1, long dim2, long nnz, long rowsPerBlock, long colsPerBlock) {
- super(l, dt, vt);
- _dataop = dop;
-
- _fileName = fname;
- setDim1(dim1);
- setDim2(dim2);
- setRowsInBlock(rowsPerBlock);
- setColsInBlock(colsPerBlock);
- setNnz(nnz);
-
- if( dop == DataOpTypes.TRANSIENTREAD )
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
- /**
- * READ operation for Matrix
- * This constructor supports expressions in parameters
- */
- public DataOp(String l, DataType dt, ValueType vt,
- DataOpTypes dop, HashMap<String, Hop> inputParameters) {
- super(l, dt, vt);
-
- _dataop = dop;
-
- int index = 0;
- for( Entry<String, Hop> e : inputParameters.entrySet() )
- {
- String s = e.getKey();
- Hop input = e.getValue();
- getInput().add(input);
- input.getParent().add(this);
-
- _paramIndexMap.put(s, index);
- index++;
- }
- if (dop == DataOpTypes.TRANSIENTREAD ){
- setInputFormatType(FileFormatTypes.BINARY);
- }
- }
-
- // WRITE operation
- // This constructor does not support any expression in parameters
- public DataOp(String l, DataType dt, ValueType vt, Hop in,
- DataOpTypes dop, String fname) {
- super(l, dt, vt);
- _dataop = dop;
- getInput().add(0, in);
- in.getParent().add(this);
- _fileName = fname;
-
- if (dop == DataOpTypes.TRANSIENTWRITE || dop == DataOpTypes.FUNCTIONOUTPUT )
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
- // CHECKPOINT operation
- // This constructor does not support any expression in parameters
- public DataOp(String l, DataType dt, ValueType vt, Hop in,
- LiteralOp level, DataOpTypes dop, String fname) {
- super(l, dt, vt);
- _dataop = dop;
- getInput().add(0, in);
- getInput().add(1, level);
- in.getParent().add(this);
- level.getParent().add(this);
- _fileName = fname;
-
- if (dop == DataOpTypes.TRANSIENTWRITE || dop == DataOpTypes.FUNCTIONOUTPUT )
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
-
- /**
- * WRITE operation for Matrix
- * This constructor supports expression in parameters
- */
- public DataOp(String l, DataType dt, ValueType vt,
- DataOpTypes dop, Hop in, HashMap<String, Hop> inputParameters) {
- super(l, dt, vt);
-
- _dataop = dop;
-
- getInput().add(0, in);
- in.getParent().add(this);
-
- if (inputParameters != null){
- int index = 1;
- for( Entry<String, Hop> e : inputParameters.entrySet() )
- {
- String s = e.getKey();
- Hop input = e.getValue();
- getInput().add(input);
- input.getParent().add(this);
-
- _paramIndexMap.put(s, index);
- index++;
- }
-
- }
-
- if (dop == DataOpTypes.TRANSIENTWRITE)
- setInputFormatType(FileFormatTypes.BINARY);
- }
-
- public DataOpTypes getDataOpType()
- {
- return _dataop;
- }
-
- public void setDataOpType( DataOpTypes type )
- {
- _dataop = type;
- }
-
- public void setOutputParams(long dim1, long dim2, long nnz, long rowsPerBlock, long colsPerBlock) {
- setDim1(dim1);
- setDim2(dim2);
- setNnz(nnz);
- setRowsInBlock(rowsPerBlock);
- setColsInBlock(colsPerBlock);
- }
-
- public void setFileName(String fn) {
- _fileName = fn;
- }
-
- public String getFileName() {
- return _fileName;
- }
-
- public int getParameterIndex(String name)
- {
- return _paramIndexMap.get(name);
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- ExecType et = optFindExecType();
- Lop l = null;
-
- // construct lops for all input parameters
- HashMap<String, Lop> inputLops = new HashMap<String, Lop>();
- for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
- inputLops.put(cur.getKey(), getInput().get(cur.getValue())
- .constructLops());
- }
-
- // Create the lop
- switch(_dataop)
- {
- case TRANSIENTREAD:
- l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null,
- getDataType(), getValueType(), true, getInputFormatType());
- setOutputDimensions(l);
- break;
-
- case PERSISTENTREAD:
- l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null,
- getDataType(), getValueType(), false, getInputFormatType());
- l.getOutputParameters().setDimensions(getDim1(), getDim2(), _inRowsInBlock, _inColsInBlock, getNnz());
- break;
-
- case PERSISTENTWRITE:
- l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null,
- getDataType(), getValueType(), false, getInputFormatType());
- ((Data)l).setExecType(et);
- setOutputDimensions(l);
- break;
-
- case TRANSIENTWRITE:
- l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null,
- getDataType(), getValueType(), true, getInputFormatType());
- setOutputDimensions(l);
- break;
-
- case FUNCTIONOUTPUT:
- l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null,
- getDataType(), getValueType(), true, getInputFormatType());
- ((Data)l).setExecType(et);
- setOutputDimensions(l);
- break;
-
- default:
- throw new LopsException("Invalid operation type for Data LOP: " + _dataop);
- }
-
- setLineNumbers(l);
- setLops(l);
-
- //add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
-
- }
-
- public void setInputFormatType(FileFormatTypes ft) {
- _inFormat = ft;
- }
-
- public FileFormatTypes getInputFormatType() {
- return _inFormat;
- }
-
- public void setInputBlockSizes( long brlen, long bclen ){
- setInputRowsInBlock(brlen);
- setInputColsInBlock(bclen);
- }
-
- public void setInputRowsInBlock( long brlen ){
- _inRowsInBlock = brlen;
- }
-
- public long getInputRowsInBlock(){
- return _inRowsInBlock;
- }
-
- public void setInputColsInBlock( long bclen ){
- _inColsInBlock = bclen;
- }
-
- public long getInputColsInBlock(){
- return _inColsInBlock;
- }
-
- public boolean isRead()
- {
- return( _dataop == DataOpTypes.PERSISTENTREAD || _dataop == DataOpTypes.TRANSIENTREAD );
- }
-
- public boolean isWrite()
- {
- return( _dataop == DataOpTypes.PERSISTENTWRITE || _dataop == DataOpTypes.TRANSIENTWRITE );
- }
-
- public boolean isPersistentReadWrite()
- {
- return( _dataop == DataOpTypes.PERSISTENTREAD || _dataop == DataOpTypes.PERSISTENTWRITE );
- }
-
- @Override
- public String getOpString() {
- String s = new String("");
- s += HopsData2String.get(_dataop);
- s += " "+getName();
- return s;
- }
-
- public void printMe() throws HopsException {
- if (LOG.isDebugEnabled()){
- if (getVisited() != VisitStatus.DONE) {
- super.printMe();
- LOG.debug(" DataOp: " + _dataop);
- if (_fileName != null) {
- LOG.debug(" file: " + _fileName);
- }
- LOG.debug(" format: " + getInputFormatType());
- for (Hop h : getInput()) {
- h.printMe();
- }
- }
- setVisited(VisitStatus.DONE);
- }
- }
-
- @Override
- public boolean allowsAllExecTypes()
- {
- return false;
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- double ret = 0;
-
- if ( getDataType() == DataType.SCALAR )
- {
- switch( getValueType() )
- {
- case INT:
- ret = OptimizerUtils.INT_SIZE; break;
- case DOUBLE:
- ret = OptimizerUtils.DOUBLE_SIZE; break;
- case BOOLEAN:
- ret = OptimizerUtils.BOOLEAN_SIZE; break;
- case STRING:
- // by default, it estimates the size of string[100]
- ret = 100 * OptimizerUtils.CHAR_SIZE; break;
- case OBJECT:
- ret = OptimizerUtils.DEFAULT_SIZE; break;
- default:
- ret = 0;
- }
- }
- else //MATRIX
- {
- if( _dataop == DataOpTypes.PERSISTENTREAD
- || _dataop == DataOpTypes.TRANSIENTREAD )
- {
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
- // output memory estimate is not required for "write" nodes (just input)
- }
-
- return ret;
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- return LocalFileUtils.BUFFER_SIZE;
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- long[] ret = null;
-
- if( _dataop == DataOpTypes.PERSISTENTWRITE
- || _dataop == DataOpTypes.TRANSIENTWRITE )
- {
- MatrixCharacteristics mc = memo.getAllInputStats(getInput().get(0));
- if( mc.dimsKnown() )
- ret = new long[]{ mc.getRows(), mc.getCols(), mc.getNonZeros() };
- }
- else if( _dataop == DataOpTypes.TRANSIENTREAD )
- {
- //prepare statistics, passed from cross-dag transient writes
- MatrixCharacteristics mc = memo.getAllInputStats(this);
- if( mc.dimsKnown() )
- ret = new long[]{ mc.getRows(), mc.getCols(), mc.getNonZeros() };
- }
-
- return ret;
- }
-
-
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- //MB: find exec type has two meanings here: (1) for write it means the actual
- //exec type, while (2) for read it affects the recompilation decision as needed
- //for example for sum(X) where the memory consumption is solely determined by the DataOp
-
- ExecType letype = (OptimizerUtils.isMemoryBasedOptLevel()) ? findExecTypeByMemEstimate() : null;
- ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
-
- //NOTE: independent of etype executed in MR (piggybacked) if input to persistent write is MR
- if( _dataop == DataOpTypes.PERSISTENTWRITE || _dataop == DataOpTypes.TRANSIENTWRITE )
- {
- checkAndSetForcedPlatform();
-
- //additional check for write only
- if( getDataType()==DataType.SCALAR )
- _etypeForced = ExecType.CP;
-
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
- }
- else
- {
- if ( OptimizerUtils.isMemoryBasedOptLevel() )
- {
- _etype = letype;
- }
- else if ( getInput().get(0).areDimsBelowThreshold() )
- {
- _etype = ExecType.CP;
- }
- else
- {
- _etype = REMOTE;
- }
-
- //check for valid CP dimensions and matrix size
- checkAndSetInvalidCPDimsAndSize();
- }
-
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
- setRequiresRecompile();
- }
- }
- else //READ
- {
- //mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && letype==REMOTE
- && (_recompileRead || _requiresCheckpoint) )
- {
- setRequiresRecompile();
- }
-
- _etype = letype;
- }
-
- return _etype;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- if( _dataop == DataOpTypes.PERSISTENTWRITE || _dataop == DataOpTypes.TRANSIENTWRITE )
- {
- Hop input1 = getInput().get(0);
- setDim1(input1.getDim1());
- setDim2(input1.getDim2());
- setNnz(input1.getNnz());
- }
- else //READ
- {
- //do nothing; dimensions updated via set output params
- }
- }
-
-
- /**
- * Explicitly disables recompilation of transient reads, this additional information
- * is required because requiresRecompile is set in a top-down manner, hence any value
- * set from a consuming operating would be overwritten by opFindExecType.
- */
- public void disableRecompileRead()
- {
- _recompileRead = false;
- }
-
-
- @Override
- @SuppressWarnings("unchecked")
- public Object clone() throws CloneNotSupportedException
- {
- DataOp ret = new DataOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._dataop = _dataop;
- ret._fileName = _fileName;
- ret._inFormat = _inFormat;
- ret._inRowsInBlock = _inRowsInBlock;
- ret._inColsInBlock = _inColsInBlock;
- ret._recompileRead = _recompileRead;
- ret._paramIndexMap = (HashMap<String, Integer>) _paramIndexMap.clone();
- //note: no deep cp of params since read-only
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java b/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java
deleted file mode 100644
index 3fe3c61..0000000
--- a/src/main/java/com/ibm/bi/dml/hops/FunctionOp.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.hops;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.lops.FunctionCallCP;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.LopProperties.ExecType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.opt.CostEstimatorHops;
-
-/**
- * This FunctionOp represents the call to a DML-bodied or external function.
- *
- * Note: Currently, we support expressions in function arguments but no function calls
- * in expressions.
- */
-public class FunctionOp extends Hop
-{
-
- public static String OPSTRING = "extfunct";
-
- public enum FunctionType{
- DML,
- EXTERNAL_MEM,
- EXTERNAL_FILE,
- MULTIRETURN_BUILTIN,
- UNKNOWN
- }
-
- private FunctionType _type = null;
- private String _fnamespace = null;
- private String _fname = null;
- private String[] _outputs = null;
- private ArrayList<Hop> _outputHops = null;
-
- private FunctionOp() {
- //default constructor for clone
- }
-
- public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs, ArrayList<Hop> outputHops) {
- this(type, fnamespace, fname, finputs, outputs);
- _outputHops = outputHops;
- }
-
- public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs)
- {
- super(fnamespace + Program.KEY_DELIM + fname, DataType.UNKNOWN, ValueType.UNKNOWN );
-
- _type = type;
- _fnamespace = fnamespace;
- _fname = fname;
- _outputs = outputs;
-
- for( Hop in : finputs )
- {
- getInput().add(in);
- in.getParent().add(this);
- }
- }
-
- public String getFunctionNamespace()
- {
- return _fnamespace;
- }
-
- public String getFunctionName()
- {
- return _fname;
- }
-
- public void setFunctionName( String fname )
- {
- _fname = fname;
- }
-
- public ArrayList<Hop> getOutputs() {
- return _outputHops;
- }
-
- public String[] getOutputVariableNames()
- {
- return _outputs;
- }
-
- public FunctionType getFunctionType()
- {
- return _type;
- }
-
- @Override
- public boolean allowsAllExecTypes() {
- return false;
- }
-
- @Override
- public void computeMemEstimate( MemoTable memo )
- {
- //overwrites default hops behavior
-
- if( _type == FunctionType.DML )
- _memEstimate = 1; //minimal mem estimate
- else if( _type == FunctionType.EXTERNAL_MEM )
- _memEstimate = 2* getInputSize(); //in/out
- else if( _type == FunctionType.EXTERNAL_FILE || _type == FunctionType.UNKNOWN )
- _memEstimate = CostEstimatorHops.DEFAULT_MEM_MR;
- else if ( _type == FunctionType.MULTIRETURN_BUILTIN ) {
- boolean outputDimsKnown = true;
- for(Hop out : getOutputs()){
- outputDimsKnown &= out.dimsKnown();
- }
- if( outputDimsKnown ) {
- long lnnz = ((_nnz>=0)?_nnz:_dim1*_dim2);
- _outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
- _processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, lnnz);
- }
- _memEstimate = getInputOutputSize();
- //System.out.println("QREst " + (_memEstimate/1024/1024));
- }
- }
-
- @Override
- protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
- {
- if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
- throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
- else {
- if ( getFunctionName().equalsIgnoreCase("qr") ) {
- // upper-triangular and lower-triangular matrices
- long outputH = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
- long outputR = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
- //System.out.println("QROut " + (outputH+outputR)/1024/1024);
- return outputH+outputR;
-
- }
- else if ( getFunctionName().equalsIgnoreCase("lu") ) {
- // upper-triangular and lower-triangular matrices
- long outputP = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0/getOutputs().get(1).getDim2());
- long outputL = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
- long outputU = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
- //System.out.println("LUOut " + (outputL+outputU+outputP)/1024/1024);
- return outputL+outputU+outputP;
-
- }
- else if ( getFunctionName().equalsIgnoreCase("eigen") ) {
- long outputVectors = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
- long outputValues = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), 1, 1.0);
- //System.out.println("EigenOut " + (outputVectors+outputValues)/1024/1024);
- return outputVectors+outputValues;
-
- }
- else
- throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
- }
- }
-
- @Override
- protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
- {
- if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
- throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
- else {
- if ( getFunctionName().equalsIgnoreCase("qr") ) {
- // matrix of size same as the input
- double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
- //System.out.println("QRInter " + interOutput/1024/1024);
- return interOutput;
- }
- else if ( getFunctionName().equalsIgnoreCase("lu")) {
- // 1D vector
- double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
- //System.out.println("LUInter " + interOutput/1024/1024);
- return interOutput;
- }
- else if ( getFunctionName().equalsIgnoreCase("eigen")) {
- // One matrix of size original input and three 1D vectors (used to represent tridiagonal matrix)
- double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0)
- + 3*OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
- //System.out.println("EigenInter " + interOutput/1024/1024);
- return interOutput;
- }
- else
- throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
- }
- }
-
- @Override
- protected long[] inferOutputCharacteristics( MemoTable memo )
- {
- throw new RuntimeException("Invalid call of inferOutputCharacteristics in FunctionOp.");
- }
-
- @Override
- public Lop constructLops()
- throws HopsException, LopsException
- {
- //return already created lops
- if( getLops() != null )
- return getLops();
-
- ExecType et = optFindExecType();
-
- if ( et != ExecType.CP ) {
- throw new HopsException("Invalid execution type for function: " + _fname);
- }
- //construct input lops (recursive)
- ArrayList<Lop> tmp = new ArrayList<Lop>();
- for( Hop in : getInput() )
- tmp.add( in.constructLops() );
-
- //construct function call
- FunctionCallCP fcall = new FunctionCallCP( tmp, _fnamespace, _fname, _outputs, _outputHops );
- setLineNumbers( fcall );
- setLops( fcall );
-
- //note: no reblock lop because outputs directly bound
-
- return getLops();
- }
-
- @Override
- public String getOpString()
- {
- return OPSTRING;
- }
-
- @Override
- protected ExecType optFindExecType()
- throws HopsException
- {
- if ( getFunctionType() == FunctionType.MULTIRETURN_BUILTIN ) {
- // Since the memory estimate is only conservative, do not throw
- // exception if the estimated memory is larger than the budget
- // Nevertheless, memory estimates these functions are useful for
- // other purposes, such as compiling parfor
- return ExecType.CP;
-
- // check if there is sufficient memory to execute this function
- /*if ( getMemEstimate() < OptimizerUtils.getMemBudget(true) ) {
- return ExecType.CP;
- }
- else {
- throw new HopsException("Insufficient memory to execute function: " + getFunctionName());
- }*/
- }
- // the actual function call is always CP
- return ExecType.CP;
- }
-
- @Override
- public void refreshSizeInformation()
- {
- //do nothing
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException
- {
- FunctionOp ret = new FunctionOp();
-
- //copy generic attributes
- ret.clone(this, false);
-
- //copy specific attributes
- ret._type = _type;
- ret._fnamespace = _fnamespace;
- ret._fname = _fname;
- ret._outputs = _outputs.clone();
-
- return ret;
- }
-
- @Override
- public boolean compare( Hop that )
- {
- return false;
- }
-}