You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2016/12/16 19:35:19 UTC
[1/3] incubator-systemml git commit: [SYSTEMML-1153] Remove unused
methods from project
Repository: incubator-systemml
Updated Branches:
refs/heads/master 6ffb6ac47 -> 1ab2d7e68
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
index 9cf0e93..7e68b61 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java
@@ -107,17 +107,6 @@ public class MatrixIndexes implements WritableComparable<MatrixIndexes>, RawComp
public String toString() {
return "("+_row+", "+_col+")";
}
-
- public int compareWithOrder(MatrixIndexes other, boolean leftcached) {
- if( !leftcached )
- return compareTo(other);
-
- if( _col != other._col)
- return (_col > other._col ? 1 : -1);
- else if( _row != other._row)
- return (_row>other._row ? 1 : -1);
- return 0;
- }
////////////////////////////////////////////////////
// implementation of Writable read/write
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
index 506a6ec..27734e1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
@@ -58,13 +58,7 @@ public class OperationsOnMatrixValues
{
valueIn.unaryOperations(op, valueOut);
}
-
- public static void performUnaryIgnoreIndexesInPlace(MatrixValue valueIn, UnaryOperator op)
- throws DMLRuntimeException
- {
- valueIn.unaryOperationsInPlace(op);
- }
-
+
public static void performReorg(MatrixIndexes indexesIn, MatrixValue valueIn, MatrixIndexes indexesOut,
MatrixValue valueOut, ReorgOperator op, int startRow, int startColumn, int length)
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockFactory.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockFactory.java
index ce99aa5..b62cff3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockFactory.java
@@ -37,10 +37,6 @@ public abstract class SparseBlockFactory
}
}
- public static SparseBlock copySparseBlock( SparseBlock.Type type, SparseBlock sblock ) {
- return copySparseBlock(type, sblock, false);
- }
-
public static SparseBlock copySparseBlock( SparseBlock.Type type, SparseBlock sblock, boolean forceCopy )
{
//sanity check for empty inputs
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
index 3cf8034..b90c5bc 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
@@ -70,14 +70,7 @@ public class SparseRow implements Serializable
values = Arrays.copyOf(that.values, cap);
indexes = Arrays.copyOf(that.indexes, cap);
}
-
- public void truncate(int newsize)
- {
- if( newsize>size || newsize<0 )
- throw new RuntimeException("truncate size: "+newsize+" should <= size: "+size+" and >=0");
- size = newsize;
- }
-
+
public int size() {
return size;
}
@@ -291,16 +284,6 @@ public class SparseRow implements Serializable
return -1;
}
- public void delete(int col)
- {
- //search for existing col index
- int index = Arrays.binarySearch(indexes, 0, size, col);
- if( index >= 0 ) {
- //shift following entries left by 1
- shiftLeftAndDelete(index);
- }
- }
-
public void deleteIndexRange(int lowerCol, int upperCol)
{
int start = searchIndexesFirstGTE(lowerCol);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
index 05ad944..4a6def4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixValue.java
@@ -39,12 +39,4 @@ public class TaggedMatrixValue extends Tagged<MatrixValue>
else
return new TaggedMatrixBlock();
}
-
- public static TaggedMatrixValue createObject(MatrixValue b, byte t)
- {
- if(b instanceof MatrixCell)
- return new TaggedMatrixCell((MatrixCell)b, t);
- else
- return new TaggedMatrixBlock((MatrixBlock)b, t);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
index 8bc69a7..3a27c0c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.lib.DelegatingMapper;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames;
@@ -59,29 +58,6 @@ public class MultipleInputs {
}
/**
- * Add a {@link Path} with a custom {@link InputFormat} and
- * {@link Mapper} to the list of inputs for the map-reduce job.
- *
- * @param conf The configuration of the job
- * @param path {@link Path} to be added to the list of inputs for the job
- * @param inputFormatClass {@link InputFormat} class to use for this path
- * @param mapperClass {@link Mapper} class to use for this path
- */
- public static void addInputPath(JobConf conf, Path path,
- Class<? extends InputFormat> inputFormatClass,
- Class<? extends Mapper> mapperClass) {
-
- addInputPath(conf, path, inputFormatClass);
-
- String mapperMapping = path.toString() + ";" + mapperClass.getName();
- String mappers = conf.get(MRConfigurationNames.MR_INPUT_MULTIPLEINPUTS_DIR_MAPPERS);
- conf.set(MRConfigurationNames.MR_INPUT_MULTIPLEINPUTS_DIR_MAPPERS, mappers == null ? mapperMapping
- : mappers + "," + mapperMapping);
-
- conf.setMapperClass(DelegatingMapper.class);
- }
-
- /**
* Retrieves a map of {@link Path}s to the {@link InputFormat} class
* that should be used for them.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
index 1a553bc..d8fa3e2 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
@@ -25,8 +25,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.OutputCollector;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -86,69 +84,6 @@ public class FrameReblockBuffer
_count++;
}
- public void appendBlock(long r_offset, long c_offset, FrameBlock inBlk, OutputCollector<Long, Writable> out )
- throws IOException
- {
- {
- int rlen = inBlk.getNumRows();
- int clen = inBlk.getNumColumns();
- for( int i=0; i<rlen; i++ )
- for( int j=0; j<clen; j++ )
- {
- Object obj = inBlk.get(i , j );
- if( obj != null )
- {
- appendCell(r_offset+i, c_offset+j, obj);
-
- //check and flush if required
- if( _count ==_bufflen )
- flushBuffer(out);
- }
- }
- }
- }
-
- public void flushBuffer( OutputCollector<Long, Writable> out )
- throws IOException
- {
- if( _count == 0 )
- return;
-
- //Step 1) sort reblock buffer (blockwise, no in-block sorting!)
- Arrays.sort( _buff, 0 ,_count, new FrameReblockBufferComparator() );
-
- //Step 2) output blocks
- Long tmpIx = -1L;
- //create intermediate blocks
- FrameBlock tmpBlock = new FrameBlock();
-
- //put values into block and output
- long cbi = -1, cbj = -1; //current block indexes
- for( int i=0; i<_count; i++ )
- {
- long bi = UtilFunctions.computeBlockIndex(_buff[i].getRow(), _brlen);
- long bj = UtilFunctions.computeBlockIndex(_buff[i].getCol(), _bclen);
-
- //output block and switch to next index pair
- if( bi != cbi || bj != cbj ) {
- outputBlock(out, tmpIx, tmpBlock);
- cbi = bi;
- cbj = bj;
- tmpIx = bi;
- tmpBlock.reset(Math.min(_brlen, (int)(_rlen-(bi-1)*_brlen)), true);
- }
-
- int ci = UtilFunctions.computeCellInBlock(_buff[i].getRow(), _brlen);
- int cj = UtilFunctions.computeCellInBlock(_buff[i].getCol(), _bclen);
- tmpBlock.set(ci, cj, _buff[i].getObjVal());
- }
-
- //output last block
- outputBlock(out, tmpIx, tmpBlock);
-
- _count = 0;
- }
-
public void flushBufferToBinaryBlocks( ArrayList<Pair<Long, FrameBlock>> outList )
throws IOException, DMLRuntimeException
{
@@ -198,17 +133,6 @@ public class FrameReblockBuffer
_count = 0;
}
- private static void outputBlock( OutputCollector<Long, Writable> out, Long key, FrameBlock block )
- throws IOException
- {
- //skip output of unassigned blocks
- if( key == -1)
- return;
-
- //output block
- out.collect(key, block);
- }
-
private static void outputBlock( ArrayList<Pair<Long, FrameBlock>> out, Long key, FrameBlock value )
throws IOException, DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
index 5057a3d..56548f5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRInputCache.java
@@ -113,13 +113,6 @@ public class MMCJMRInputCache extends MMCJMRCache
return _buffer[lpos];
}
- public void resetCache()
- throws IOException
- {
- //by default don't reset buffersize (e.g., for aggregator)
- resetCache(false);
- }
-
public void resetCache(boolean fullreset)
throws IOException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
index 0745afc..7867132 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@@ -83,7 +82,6 @@ import org.apache.sysml.runtime.matrix.data.IdenticalConverter;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.MatrixValue;
import org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -266,39 +264,7 @@ public class MRJobConfiguration
{
return job.getInt(MRConfigurationNames.IO_FILE_BUFFER_SIZE, 4096);
}
-
- public static final int getJVMMaxMemSize(JobConf job)
- {
- String str=job.get(MRConfigurationNames.MR_CHILD_JAVA_OPTS);
- int start=str.indexOf("-Xmx");
- if(start<0)
- return 209715200; //default 200MB
- str=str.substring(start+4);
- int i=0;
- for(; i<str.length() && str.charAt(i)<='9' && str.charAt(i)>='0'; i++);
- int ret=Integer.parseInt(str.substring(0, i));
- if(i>=str.length())
- return ret;
-
- switch(str.charAt(i))
- {
- case 'k':
- case 'K':
- ret=ret*1024;
- break;
- case 'm':
- case 'M':
- ret=ret*1048576;
- break;
- case 'g':
- case 'G':
- ret=ret*1073741824;
- break;
- default:
- }
- return ret;
- }
-
+
public static void setMMCJCacheSize(JobConf job, long size)
{
job.setLong(MMCJ_CACHE_SIZE, size);
@@ -447,13 +413,6 @@ public class MRJobConfiguration
return job.get(MRConfigurationNames.MR_JOBTRACKER_STAGING_ROOT_DIR);
}
- public static void setStagingDir( JobConf job )
- {
- String dir = DMLConfig.LOCAL_MR_MODE_STAGING_DIR +
- Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR;
- job.set( MRConfigurationNames.MR_JOBTRACKER_STAGING_ROOT_DIR, dir );
- }
-
public static void setInputInfo(JobConf job, byte input, InputInfo inputinfo,
int brlen, int bclen, ConvertTarget target)
{
@@ -522,21 +481,7 @@ public class MRJobConfiguration
}
return outputConverter;
}
-
- @SuppressWarnings("unchecked")
- public static Class<Writable> getInputKeyClass(JobConf job, byte input)
- {
- return (Class<Writable>) job.getClass(INPUT_KEY_CLASS_PREFIX_CONFIG+input,
- MatrixIndexes.class);
- }
-
- @SuppressWarnings("unchecked")
- public static Class<Writable> getInputValueClass(JobConf job, byte input)
- {
- return (Class<Writable>) job.getClass(INPUT_VALUE_CLASS_PREFIX_CONFIG+input,
- DoubleWritable.class);
- }
-
+
public static MRInstruction[] getInstructionsInReducer(JobConf job) throws DMLRuntimeException
{
String str=job.get(INSTRUCTIONS_IN_REDUCER_CONFIG);
@@ -756,12 +701,7 @@ public class MRJobConfiguration
{
return InputInfo.stringToInputInfo( job.get(RESULTMERGE_INPUT_INFO_CONFIG) );
}
-
- public static String getResultMergeStagingDir( JobConf job )
- {
- return job.get(RESULTMERGE_STAGING_DIR_CONFIG) + job.get(MRConfigurationNames.MR_TASK_ID);
- }
-
+
public static long[] getResultMergeMatrixCharacteristics( JobConf job )
{
long[] ret = new long[4];
@@ -1039,12 +979,7 @@ public class MRJobConfiguration
{
return job.getLong(INPUT_MATRIX_NUM_NNZ_PREFIX_CONFIG+matrixIndex, 1);
}
-
- public static void handleRecordReaderInstrucion(JobConf job, String recordReaderInstruction, String[] inputs, InputInfo[] inputInfos)
- {
- //do nothing, not used currently
- }
-
+
public static void setupDistCacheInputs(JobConf job, String indices, String pathsString, ArrayList<String> paths) {
job.set(DISTCACHE_INPUT_INDICES, indices);
job.set(DISTCACHE_INPUT_PATHS, pathsString);
@@ -1061,11 +996,7 @@ public class MRJobConfiguration
public static String getDistCacheInputIndices(JobConf job) {
return job.get(DISTCACHE_INPUT_INDICES);
}
-
- public static String getDistCacheInputPaths(JobConf job) {
- return job.get(DISTCACHE_INPUT_PATHS);
- }
-
+
private static String getCSVString(PDataPartitionFormat[] formats) {
if ( formats == null || formats.length == 0 )
return "";
@@ -1228,12 +1159,7 @@ public class MRJobConfiguration
paths.add(p);
}
}
-
-
- public static void updateResultDimsUnknown (JobConf job, byte[] updDimsUnknown) {
- job.set(RESULT_DIMS_UNKNOWN_CONFIG, MRJobConfiguration.getIndexesString(updDimsUnknown));
- }
-
+
public static void setUpMultipleOutputs(JobConf job, byte[] resultIndexes, byte[] resultDimsUnknown, String[] outputs,
OutputInfo[] outputInfos, boolean inBlockRepresentation, boolean mayContainCtable)
throws Exception
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
index a70a190..878c236 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java
@@ -40,7 +40,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.MatrixValue;
import org.apache.sysml.runtime.matrix.data.Pair;
-import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
@SuppressWarnings("rawtypes")
public abstract class MapperBase extends MRBaseForCommonInstructions
@@ -398,30 +397,4 @@ public abstract class MapperBase extends MRBaseForCommonInstructions
throw new IOException(e);
}
}
-
- protected void processMapOutputToReducer(int index, MatrixIndexes indexBuffer,
- TaggedMatrixValue taggedValueBuffer, OutputCollector<Writable, Writable> out) throws IOException
- {
-
- for(byte output: outputIndexes.get(index))
- {
- ArrayList<IndexedMatrixValue> results= cachedValues.get(output);
- if(results==null)
- continue;
- for(IndexedMatrixValue result: results)
- {
- if(result==null)
- continue;
- indexBuffer.setIndexes(result.getIndexes());
- ////////////////////////////////////////
- // taggedValueBuffer.getBaseObject().copy(result.getValue());
- taggedValueBuffer.setBaseObject(result.getValue());
- ////////////////////////////////////////
- taggedValueBuffer.setTag(output);
- out.collect(indexBuffer, taggedValueBuffer);
- // System.out.println("map output: "+indexBuffer+"\n"+taggedValueBuffer);
- }
-
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
index 6b95753..dcec043 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/PartialAggregator.java
@@ -132,13 +132,6 @@ public class PartialAggregator extends MMCJMRCache
return nonZeros;
}
- public void close()
- throws IOException
- {
- if( !memOnly )
- super.deleteAllWorkingFiles();
- }
-
private void aggregateToBufferHelp(MatrixIndexes indexes, MatrixValue value, boolean leftcached)
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
index 34d9727..6b03656 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/CompactOutputFormat.java
@@ -39,16 +39,6 @@ public class CompactOutputFormat<K extends Writable, V extends Writable> extends
static final String FINAL_SYNC_ATTRIBUTE = "final.sync";
- /**
- * Set the requirement for a final sync before the stream is closed.
- *
- * @param conf job configuration
- * @param newValue true if do a final sync before the stream is closed
- */
- public static void setFinalSync(JobConf conf, boolean newValue) {
- conf.setBoolean(FINAL_SYNC_ATTRIBUTE, newValue);
- }
-
/**
* Does the user want a final sync at close?
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
index 3e38d6e..cb35aec 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/TfUtils.java
@@ -246,14 +246,6 @@ public class TfUtils implements Serializable{
_da = new DummycodeAgent(spec, _outputColumnNames, _numInputCols);
}
- public void setupAgents(OmitAgent oa, MVImputeAgent mia, RecodeAgent ra, BinAgent ba, DummycodeAgent da) {
- _oa = oa;
- _mia = mia;
- _ra = ra;
- _ba = ba;
- _da = da;
- }
-
private void parseColumnNames() {
_outputColumnNames = _delim.split(_headerLine, -1);
for(int i=0; i < _outputColumnNames.length; i++)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
index a7ff9fa..c868f60 100644
--- a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
@@ -88,28 +88,7 @@ public class LocalFileUtils
public static MatrixBlock readMatrixBlockFromLocal(String filePathAndName, MatrixBlock reuse) throws IOException {
return (MatrixBlock) readWritableFromLocal(filePathAndName, reuse);
}
-
- /** Reads a frame block from local file system.
- *
- * @param filePathAndName file to read
- * @return frame block
- * @throws IOException if IOException occurs
- */
- public static FrameBlock readFrameBlockFromLocal(String filePathAndName) throws IOException {
- return (FrameBlock) readWritableFromLocal(filePathAndName, new FrameBlock());
- }
-
- /** Reads a frame block from local file system.
- *
- * @param filePathAndName file to read
- * @param reuse frame block to reuse
- * @return frame block
- * @throws IOException if IOException occurs
- */
- public static FrameBlock readFrameBlockFromLocal(String filePathAndName, FrameBlock reuse) throws IOException {
- return (FrameBlock) readWritableFromLocal(filePathAndName, reuse);
- }
-
+
/** Reads a matrix/frame block from local file system.
*
* @param filePathAndName file to read
@@ -157,17 +136,7 @@ public class LocalFileUtils
public static void writeMatrixBlockToLocal(String filePathAndName, MatrixBlock mb) throws IOException {
writeWritableToLocal(filePathAndName, mb);
}
-
- /** Writes a matrix block to local file system.
- *
- * @param filePathAndName file to write
- * @param fb frame block
- * @throws IOException if IOException occurs
- */
- public static void writeFrameBlockToLocal(String filePathAndName, FrameBlock fb) throws IOException {
- writeWritableToLocal(filePathAndName, fb);
- }
-
+
/** Writes a matrix/frame block to local file system.
*
* @param filePathAndName file to write
@@ -433,14 +402,6 @@ public class LocalFileUtils
return count;
}
- public static String getWorkingDir()
- throws DMLRuntimeException
- {
- if( _workingDir == null )
- createWorkingDirectory();
- return _workingDir;
- }
-
public static String getWorkingDir( String category )
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
index 3a37377..6f083f7 100644
--- a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
+++ b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
@@ -84,21 +84,7 @@ public class MapReduceTool
// remove all the leading 0s
return String.valueOf(Long.parseLong(nodePrefix));
}
-
- @Deprecated
- public static String getUniqueKeyPerTaskWithLeadingZros(JobConf job, boolean inMapper) {
- String nodePrefix = job.get(MRConfigurationNames.MR_TASK_ATTEMPT_ID);
- int i;
- if (inMapper)
- i = nodePrefix.indexOf("_m_");
- else
- i = nodePrefix.indexOf("_r_");
- int j = nodePrefix.lastIndexOf("_");
- nodePrefix = nodePrefix.substring(i + 3, j);
- return nodePrefix;
- }
-
public static int getUniqueTaskId(JobConf job) {
//TODO: investigate ID pattern, required for parallel jobs
/*String nodePrefix = job.get(MRConfigurationNames.MR_TASK_ATTEMPT_ID);
@@ -156,13 +142,6 @@ public class MapReduceTool
}
}
- public static boolean isHDFSDirectory(String dir) throws IOException {
- FileSystem fs = FileSystem.get(_rJob);
- Path pth = new Path(dir);
- FileStatus fstat = fs.getFileStatus(pth);
- return fstat.isDirectory();
- }
-
public static boolean isHDFSFileEmpty(String dir) throws IOException {
FileSystem fs = FileSystem.get(_rJob);
return isFileEmpty(fs, dir);
@@ -226,37 +205,6 @@ public class MapReduceTool
}
}
- public static String getSubDirs(String dir)
- throws IOException
- {
- FileSystem fs = FileSystem.get(_rJob);
- FileStatus[] files = fs.listStatus(new Path(dir));
- StringBuilder sb = new StringBuilder();
- for (FileStatus file : files) {
- if ( sb.length()>0 )
- sb.append(",");
- sb.append(file.getPath().toString());
- }
- return sb.toString();
- }
-
- public static String getSubDirsIgnoreLogs(String dir)
- throws IOException
- {
- FileSystem fs = FileSystem.get(_rJob);
- FileStatus[] files = fs.listStatus(new Path(dir));
- StringBuilder sb = new StringBuilder();
- for (FileStatus file : files) {
- String name = file.getPath().toString();
- if (name.contains("_logs"))
- continue;
- if( sb.length()>0 )
- sb.append(",");
- sb.append(name);
- }
- return sb.toString();
- }
-
/**
* Returns the size of a file or directory on hdfs in bytes.
*
@@ -622,13 +570,6 @@ public class MapReduceTool
return new double[] {ret, (average ? -1 : readValue.get()), (average ? -1 : cum_weight)};
}
- public static int extractNumberFromOutputFile(String name)
- {
- int i=name.indexOf("part-");
- assert(i>=0);
- return Integer.parseInt(name.substring(i+5));
- }
-
public static void createDirIfNotExistOnHDFS(String dir, String permissions)
throws IOException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/RandNPair.java b/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
index 81f3176..988df3a 100644
--- a/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
+++ b/src/main/java/org/apache/sysml/runtime/util/RandNPair.java
@@ -45,9 +45,5 @@ public class RandNPair
public void compute(Random r) {
compute(r.nextDouble(), r.nextDouble());
}
-
- public void compute(long seed) {
- compute(new Random(seed));
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java b/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
index 5f18f37..529ca03 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UniformPRNGenerator.java
@@ -44,8 +44,4 @@ public class UniformPRNGenerator extends PRNGenerator {
public double nextDouble() {
return runif.nextDouble();
}
-
- public int nextInt(int n) {
- return runif.nextInt(n);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index a02e20a..e17eb14 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -24,16 +24,12 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.ArrayUtils;
-import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
-import org.apache.sysml.runtime.instructions.InstructionUtils;
-import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.Pair;
import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
-import org.apache.wink.json4j.JSONArray;
public class UtilFunctions
{
@@ -50,12 +46,7 @@ public class UtilFunctions
public static int longHashCode(long v) {
return (int)(v^(v>>>32));
}
-
- @Deprecated
- public static int longlongHashCodeOld(long key1, long key2) {
- return UtilFunctions.longHashCode((key1<<32)+key2+ADD_PRIME1)%DIVIDE_PRIME;
- }
-
+
/**
* Returns the hash code for a long-long pair. This is the default
* hash function for the keys of a distributed matrix in MR/Spark.
@@ -130,17 +121,6 @@ public class UtilFunctions
long remain = len - (blockIndex-1)*blockSize;
return (int)Math.min(blockSize, remain);
}
-
- //all boundaries are inclusive
- public static boolean isOverlap(long s1, long f1, long s2, long f2)
- {
- return !(f2<s1 || f1<s2);
- }
-
- public static boolean isIn(long point, long s, long f)
- {
- return (point>=s && point<=f);
- }
public static boolean isInBlockRange( MatrixIndexes ix, int brlen, int bclen, long rl, long ru, long cl, long cu )
{
@@ -436,17 +416,7 @@ public class UtilFunctions
return false;
return true;
}
-
- public static boolean isSimpleDoubleNumber( String str )
- {
- //true if all chars numeric or - or .
- byte[] c = str.getBytes();
- for( int i=0; i<c.length; i++ )
- if( (c[i] < 48 || c[i] > 57) && !(c[i]==45 || c[i]==46) )
- return false;
- return true;
- }
-
+
public static byte max( byte[] array )
{
byte ret = Byte.MIN_VALUE;
@@ -468,16 +438,6 @@ public class UtilFunctions
return "\"" + s + "\"";
}
- public static String toString(int[] list) {
- StringBuilder sb = new StringBuilder();
- sb.append(list[0]);
- for(int i=1; i<list.length; i++) {
- sb.append(",");
- sb.append(list[i]);
- }
- return sb.toString();
- }
-
/**
* Parses a memory size with optional g/m/k quantifiers into its
* number representation.
@@ -529,37 +489,6 @@ public class UtilFunctions
return ret;
}
- /**
- * Returns the schema based on Json object
- *
- * @param schemaObject schema object
- * @return schema as a list of value types
- */
- public static List<ValueType> getSchemaType(Object schemaObject)
- {
- JSONArray schemaJsonArr = (JSONArray)schemaObject;
- ValueType[] schemaArray = new ValueType[schemaJsonArr.size()];
-
- for(int i=0; i < schemaJsonArr.length(); i++)
- schemaArray[i] = ValueType.valueOf((String)schemaJsonArr.get(0));
- return Arrays.asList(schemaArray);
- }
-
- /**
- * This function will return datatype, if its Matrix or Frame
- *
- * @param str Instruction string to execute
- * @param index parts index
- * @return data type
- */
- public static DataType getDataType(String str, int index)
- {
- String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
- CPOperand in1 = new CPOperand(parts[index]);
-
- return in1.getDataType();
- }
-
public static double getDouble(Object obj) {
return (obj instanceof Double) ? (Double)obj :
Double.parseDouble(obj.toString());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java b/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
index 4896658..946226e 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
@@ -19,9 +19,6 @@
package org.apache.sysml.yarn.ropt;
-import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.hops.OptimizerUtils;
-
public class YarnOptimizerUtils
{
public enum GridEnumType{
@@ -31,43 +28,6 @@ public class YarnOptimizerUtils
HYBRID_MEM_EXP_GRID,
}
- public static double getRemoteMemBudgetMap(long jobLookupId)
- {
- return getRemoteMemBudgetMap(false, jobLookupId);
- }
-
- public static double getRemoteMemBudgetMap(boolean substractSortBuffer, long jobLookupId)
- {
- double ret = YarnClusterAnalyzer.getRemoteMaxMemoryMap(jobLookupId);
- if( substractSortBuffer )
- ret -= YarnClusterAnalyzer.getRemoteMaxMemorySortBuffer();
- return ret * OptimizerUtils.MEM_UTIL_FACTOR;
- }
-
- public static double getRemoteMemBudgetReduce(long jobLookupId)
- {
- double ret = YarnClusterAnalyzer.getRemoteMaxMemoryReduce(jobLookupId);
- return ret * OptimizerUtils.MEM_UTIL_FACTOR;
- }
-
- /**
- * Returns the number of reducers that potentially run in parallel.
- * This is either just the configured value (SystemML config) or
- * the minimum of configured value and available reduce slots.
- *
- * @param configOnly if true, return number or reducers from SystemML configuration
- * @param jobLookupId hadoop job id
- * @return number of reducers
- */
- public static int getNumReducers(boolean configOnly, long jobLookupId)
- {
- int ret = ConfigurationManager.getNumReducers();
- if( !configOnly )
- ret = Math.min(ret,YarnClusterAnalyzer.getRemoteParallelReduceTasks(jobLookupId));
-
- return ret;
- }
-
public static long toB( long mb )
{
return 1024 * 1024 * mb;
[3/3] incubator-systemml git commit: [SYSTEMML-1153] Remove unused
methods from project
Posted by de...@apache.org.
[SYSTEMML-1153] Remove unused methods from project
Remove selected unreferenced methods, which removes ~3300 LOC from 98 files.
Closes #316.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/1ab2d7e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/1ab2d7e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/1ab2d7e6
Branch: refs/heads/master
Commit: 1ab2d7e68344f7fb30933eacd8520261e1c6c5b0
Parents: 6ffb6ac
Author: Deron Eriksson <de...@us.ibm.com>
Authored: Fri Dec 16 11:27:52 2016 -0800
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Fri Dec 16 11:27:52 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/sysml/api/DMLScript.java | 19 --
.../api/monitoring/SparkMonitoringUtil.java | 15 +-
.../java/org/apache/sysml/conf/DMLConfig.java | 54 +----
.../sysml/debug/DMLBreakpointManager.java | 86 +-------
.../sysml/debug/DMLDebuggerInterface.java | 6 -
.../java/org/apache/sysml/debug/DebugState.java | 14 +-
.../org/apache/sysml/hops/ConvolutionOp.java | 55 +----
src/main/java/org/apache/sysml/hops/Hop.java | 125 +----------
.../org/apache/sysml/hops/OptimizerUtils.java | 44 +---
.../sysml/hops/cost/CostEstimationWrapper.java | 20 --
.../apache/sysml/hops/cost/CostEstimator.java | 25 +--
.../sysml/hops/recompile/RecompileStatus.java | 17 +-
.../apache/sysml/hops/recompile/Recompiler.java | 7 +-
.../sysml/hops/rewrite/HopRewriteUtils.java | 27 +--
src/main/java/org/apache/sysml/lops/Lop.java | 4 -
.../org/apache/sysml/lops/PartialAggregate.java | 7 +-
.../java/org/apache/sysml/lops/Ternary.java | 15 --
.../java/org/apache/sysml/lops/Transform.java | 26 ---
.../java/org/apache/sysml/lops/compile/Dag.java | 8 +-
.../org/apache/sysml/lops/compile/JobType.java | 16 --
.../sysml/parser/ConditionalPredicate.java | 19 +-
.../org/apache/sysml/parser/DMLProgram.java | 10 +-
.../org/apache/sysml/parser/DMLTranslator.java | 9 -
.../org/apache/sysml/parser/DataIdentifier.java | 33 ---
.../apache/sysml/parser/FunctionStatement.java | 22 +-
.../org/apache/sysml/parser/IfStatement.java | 32 +--
.../apache/sysml/parser/IndexedIdentifier.java | 29 ---
.../sysml/parser/LiveVariableAnalysis.java | 5 -
.../apache/sysml/parser/OutputStatement.java | 4 -
.../ParameterizedBuiltinFunctionExpression.java | 6 +-
.../java/org/apache/sysml/parser/Statement.java | 20 --
.../org/apache/sysml/parser/StatementBlock.java | 16 +-
.../org/apache/sysml/parser/VariableSet.java | 7 +-
.../parser/common/CommonSyntacticValidator.java | 17 --
.../parser/common/CustomErrorListener.java | 8 -
.../sysml/runtime/compress/ColGroupBitmap.java | 14 +-
.../runtime/compress/PlanningBinPacker.java | 59 -----
.../estim/CompressedSizeEstimatorSample.java | 4 -
.../runtime/compress/utils/ConverterUtils.java | 16 --
.../compress/utils/LinearAlgebraUtils.java | 164 --------------
.../ExternalFunctionProgramBlock.java | 25 ---
.../runtime/controlprogram/ForProgramBlock.java | 6 +-
.../controlprogram/LocalVariableMap.java | 44 +---
.../runtime/controlprogram/ProgramBlock.java | 10 +-
.../controlprogram/WhileProgramBlock.java | 6 +-
.../controlprogram/caching/CacheStatistics.java | 19 +-
.../context/ExecutionContext.java | 26 +--
.../context/SparkExecutionContext.java | 12 --
.../controlprogram/parfor/LocalParWorker.java | 10 +-
.../controlprogram/parfor/LocalTaskQueue.java | 12 --
.../controlprogram/parfor/ParWorker.java | 6 -
.../controlprogram/parfor/ProgramConverter.java | 17 --
.../runtime/controlprogram/parfor/Task.java | 18 --
.../parfor/opt/CostEstimator.java | 55 -----
.../parfor/opt/CostEstimatorRuntime.java | 13 --
.../controlprogram/parfor/opt/OptNode.java | 57 +----
.../parfor/opt/OptTreeConverter.java | 119 -----------
.../parfor/opt/OptTreePlanMappingRuntime.java | 11 +-
.../parfor/opt/OptimizationWrapper.java | 141 +-----------
.../controlprogram/parfor/opt/Optimizer.java | 60 ------
.../parfor/opt/OptimizerRuleBased.java | 44 +---
.../controlprogram/parfor/opt/PerfTestTool.java | 95 +--------
.../parfor/opt/ProgramRecompiler.java | 161 +-------------
.../parfor/stat/InfrastructureAnalyzer.java | 29 +--
.../parfor/stat/StatisticMonitor.java | 12 +-
.../controlprogram/parfor/util/IDHandler.java | 32 ---
.../sysml/runtime/functionobjects/Builtin.java | 44 +---
.../functionobjects/OffsetColumnIndex.java | 7 +-
.../runtime/instructions/InstructionUtils.java | 24 +--
.../runtime/instructions/MRJobInstruction.java | 213 -------------------
.../runtime/instructions/cp/CPOperand.java | 17 +-
.../instructions/cp/VariableCPInstruction.java | 20 --
...ReturnParameterizedBuiltinSPInstruction.java | 4 -
.../instructions/spark/UnarySPInstruction.java | 10 -
.../spark/utils/RDDAggregateUtils.java | 106 ---------
.../instructions/spark/utils/SparkUtils.java | 202 +-----------------
.../sysml/runtime/io/FrameReaderFactory.java | 14 --
.../apache/sysml/runtime/io/WriterTextCSV.java | 86 --------
.../runtime/matrix/data/LibMatrixReorg.java | 17 +-
.../runtime/matrix/data/MatrixIndexes.java | 11 -
.../matrix/data/OperationsOnMatrixValues.java | 8 +-
.../runtime/matrix/data/SparseBlockFactory.java | 4 -
.../sysml/runtime/matrix/data/SparseRow.java | 19 +-
.../runtime/matrix/data/TaggedMatrixValue.java | 8 -
.../matrix/data/hadoopfix/MultipleInputs.java | 24 ---
.../matrix/mapred/FrameReblockBuffer.java | 76 -------
.../runtime/matrix/mapred/MMCJMRInputCache.java | 7 -
.../matrix/mapred/MRJobConfiguration.java | 86 +-------
.../sysml/runtime/matrix/mapred/MapperBase.java | 27 ---
.../matrix/mapred/PartialAggregator.java | 7 -
.../matrix/sort/CompactOutputFormat.java | 10 -
.../apache/sysml/runtime/transform/TfUtils.java | 8 -
.../sysml/runtime/util/LocalFileUtils.java | 43 +---
.../sysml/runtime/util/MapReduceTool.java | 59 -----
.../apache/sysml/runtime/util/RandNPair.java | 4 -
.../sysml/runtime/util/UniformPRNGenerator.java | 4 -
.../sysml/runtime/util/UtilFunctions.java | 75 +------
.../sysml/yarn/ropt/YarnOptimizerUtils.java | 40 ----
98 files changed, 76 insertions(+), 3332 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index 3df1320..bc6bacb 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -227,25 +227,6 @@ public class DMLScript
}
}
- public static boolean executeScript( String[] args )
- throws DMLException
- {
- Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf());
- return executeScript( conf, args );
- }
-
- public static String executeScript( Configuration conf, String[] args, boolean suppress)
- throws DMLException
- {
- _suppressPrint2Stdout = suppress;
- try {
- boolean ret = executeScript(conf, args);
- return Boolean.toString(ret);
- } catch(DMLScriptException e) {
- return e.getMessage();
- }
- }
-
/**
* Single entry point for all public invocation alternatives (e.g.,
* main, executeScript, JaqlUdf etc)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java b/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
index 82a0fa2..42a1f32 100644
--- a/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
+++ b/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java
@@ -263,20 +263,7 @@ public class SparkMonitoringUtil {
}
}
-
- /**
- * Useful to avoid passing large String through Py4J
- * @param fileName the file name
- * @throws DMLRuntimeException if DMLRuntimeException occurs
- * @throws IOException if IOException occurs
- */
- public void saveRuntimeInfoInJSONFormat(String fileName) throws DMLRuntimeException, IOException {
- String json = getRuntimeInfoInJSONFormat();
- BufferedWriter bw = new BufferedWriter(new FileWriter(fileName));
- bw.write(json);
- bw.close();
- }
-
+
public String getRuntimeInfoInJSONFormat() throws DMLRuntimeException, IOException {
StringBuilder retVal = new StringBuilder("{\n");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index 4a70313..ec3a30d 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -145,41 +145,7 @@ public class DMLConfig
{
_xmlRoot = root;
}
-
- public void merge(DMLConfig otherConfig)
- throws ParseException
- {
- if (otherConfig == null)
- return;
-
- try {
- // for each element in otherConfig, either overwrite existing value OR add to defaultConfig
- NodeList otherConfigNodeList = otherConfig._xmlRoot.getChildNodes();
- if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
- for (int i=0; i<otherConfigNodeList.getLength(); i++){
- org.w3c.dom.Node optionalConfigNode = otherConfigNodeList.item(i);
-
- if (optionalConfigNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE){
-
- // try to find optional config node in default config node
- String paramName = optionalConfigNode.getNodeName();
- String paramValue = ((Element)optionalConfigNode).getFirstChild().getNodeValue();
-
- if (_xmlRoot.getElementsByTagName(paramName) != null)
- LOG.info("Updating " + paramName + " with value " + paramValue);
- else
- LOG.info("Defining new attribute" + paramName + " with value " + paramValue);
- DMLConfig.setTextValue(_xmlRoot, paramName, paramValue);
- }
-
- }
- } // end if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
- } catch (Exception e){
- LOG.error("Failed in merge default config file with optional config file",e);
- throw new ParseException("ERROR: error merging config file " + otherConfig._fileName + " with " + _fileName);
- }
- }
-
+
/**
* Method to parse configuration
* @throws ParserConfigurationException
@@ -458,23 +424,7 @@ public class DMLConfig
elem.getFirstChild().setNodeValue(String.valueOf(mrMem));
}
}
-
- @SuppressWarnings("deprecation")
- public void makeQualifiedScratchSpacePath()
- throws IOException
- {
- NodeList list2 = _xmlRoot.getElementsByTagName(SCRATCH_SPACE);
- if (list2 != null && list2.getLength() > 0) {
- Element elem = (Element) list2.item(0);
-
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- String fname = elem.getFirstChild().getNodeValue();
- Path path = new Path(fname).makeQualified(fs);
-
- elem.getFirstChild().setNodeValue(path.toString());
- }
- }
-
+
public static String getDefaultTextValue( String key ) {
return _defaultVals.get( key );
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java b/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
index 4151398..2ec3744 100644
--- a/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
+++ b/src/main/java/org/apache/sysml/debug/DMLBreakpointManager.java
@@ -46,20 +46,6 @@ public class DMLBreakpointManager {
}
/**
- * Returns size of active DML breakpoints
- * @return size Size of active breakpoints
- */
- public static int getBreakpointsSize()
- {
- int size = 0;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (breakpoints.get(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE)
- size++;
- }
- return size;
- }
-
- /**
* Returns breakpoint instruction at a particular line number (if any)
* @param lineNumber Location of breakpoint
* @return Breakpoint instruction at indicated line number (if any)
@@ -71,52 +57,6 @@ public class DMLBreakpointManager {
}
/**
- * Returns breakpoint instruction with given breakpoint id
- * @param location Breakpoint id
- * @return Breakpoint instruction at indicated id
- */
- public static BreakPointInstruction getBreakpointAtIndex(int location) {
- int index = 1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (index++ == location) {
- return breakpoints.get(lineNumber);
- }
- }
- return null;
- }
-
- /**
- * Returns breakpoint line number with given breakpoint id
- * @param location Breakpoint id
- * @return Breakpoint instruction line number (-1 if not found)
- */
- public static int getBreakpointLineNumber(int location) {
- int index = 1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (index++ == location) {
- return lineNumber;
- }
- }
- return -1;
- }
-
- /**
- * Returns breakpoint identifier with given line number
- * @param lineNum Line number Location of breakpoint in DML script
- * @return Breakpoint id within all breakpoints (-1 if not found)
- */
- public static int getBreakpointID(int lineNum) {
- int bpID=1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (lineNum == lineNumber) {
- return bpID;
- }
- bpID++;
- }
- return -1;
- }
-
- /**
* Insert a breakpoint instruction into list of existing breakpoints.
*
* @param breakpoint the breakpoint instruction
@@ -156,20 +96,7 @@ public class DMLBreakpointManager {
System.out.format("Breakpoint updated at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
}
}
-
- /**
- * Updates breakpoint status for a given breakpoint id
- * @param location Breakpoint identifier
- * @param status Current breakpoint status
- */
- public static void updateBreakpointID(int location, BPINSTRUCTION_STATUS status) {
- int lineNumber = getBreakpointLineNumber(location);
- if (lineNumber != -1) {
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- System.out.format("Breakpoint updated at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
- }
- }
-
+
/**
* Removes breakpoint instruction at given line number
* @param lineNumber Location for inserting breakpoint
@@ -181,15 +108,4 @@ public class DMLBreakpointManager {
System.out.format("Breakpoint deleted at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
}
}
-
- /**
- * Removes breakpoint instruction at given location
- * @param location Breakpoint instruction id
- * @param status Current breakpoint status
- */
- public static void removeBreakpointIndex(int location, BPINSTRUCTION_STATUS status) {
- int lineNumber = getBreakpointLineNumber(location);
- if (lineNumber != -1)
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java b/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
index f5f22cd..0750562 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDebuggerInterface.java
@@ -179,12 +179,6 @@ public class DMLDebuggerInterface
System.out.flush();
}
- public void writeToStandardError(String errStr) {
- System.err.print(errStr);
- System.err.flush();
- }
-
-
public void writelnToStandardError(String errStr) {
System.err.println(errStr);
System.err.flush();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/debug/DebugState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/debug/DebugState.java b/src/main/java/org/apache/sysml/debug/DebugState.java
index 80b14e7..0294d7f 100644
--- a/src/main/java/org/apache/sysml/debug/DebugState.java
+++ b/src/main/java/org/apache/sysml/debug/DebugState.java
@@ -154,19 +154,7 @@ public class DebugState
return null;
return callStack.pop();
}
-
- /**
- * Get stack frame at indicated location (if any)
- * @param location Frame position in call stack
- * @return Stack frame at specified location
- */
- protected DMLFrame getFrame(int location) {
- if (location < 0 || location >= callStack.size()) {
- return null;
- }
- return callStack.elementAt(location);
- }
-
+
/**
* Get current call stack (if any)
* @return Stack callStack
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
index 1477139..a00feb7 100644
--- a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
+++ b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
@@ -119,51 +119,7 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop
public void setOp(ConvOp op) {
this.op = op;
}
-
- public static Lop constructFusedConvolutionLops(ExecType et,
- ArrayList<Hop> inputs,
- ConvOp op, ConvolutionOp primaryOp,
- long rlen, long clen) throws HopsException, LopsException {
- int expectedNumInputs = 13;
- if(op == ConvOp.MAX_POOLING_BACKWARD
- || op == ConvOp.DIRECT_CONV2D
- || op == ConvOp.DIRECT_CONV2D_BACKWARD_FILTER
- || op == ConvOp.DIRECT_CONV2D_BACKWARD_DATA) {
- expectedNumInputs = 14;
- }
-
- if(inputs.size() != expectedNumInputs) {
- throw new HopsException("Incorrect number of inputs for " + op.name());
- }
-
- Lop in = inputs.get(0).constructLops();
- int numThreads = et == ExecType.CP ? OptimizerUtils.getConstrainedNumThreads(primaryOp.getMaxNumThreads()) : 1;
- ConvolutionTransform transform1 = new ConvolutionTransform( in,
- HopsConv2Lops.get(op), primaryOp.getDataType(), primaryOp.getValueType(), et, numThreads);
-
- // setOutputDimensions(transform1);
- transform1.getOutputParameters().setDimensions(
- rlen, clen, primaryOp.getRowsInBlock(), primaryOp.getColsInBlock(), -1, primaryOp.getUpdateType());
-
- // setLineNumbers(transform1);
- transform1.setAllPositions(primaryOp.getBeginLine(), primaryOp.getBeginColumn(), primaryOp.getEndLine(), primaryOp.getEndColumn());
-
- in.addOutput(transform1);
-
- // stride1, stride2, padding1, padding2
- // input_shape1, input_shape2, input_shape3, input_shape4,
- // filter_shape1, filter_shape2, filter_shape3, filter_shape4
- for( int i=1; i < inputs.size(); i++ )
- {
- Lop ltmp = inputs.get(i).constructLops();
- transform1.addInput(ltmp);
- //if(i == 1 && expectedNumInputs == 14)
- ltmp.addOutput(transform1);
- }
- transform1.setLevel(); //force order of added lops
- return transform1;
- }
-
+
public Lop constructConvolutionLops(ExecType et, ArrayList<Hop> inputs) throws HopsException, LopsException {
int expectedNumInputs = 13;
if(op == ConvOp.MAX_POOLING_BACKWARD
@@ -383,14 +339,7 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop
}
return params;
}
-
- long getExtractedVal(long val1, long val2) {
- if(val1 == -1 || val2 == -1) {
- return -1;
- }
- return val1*val2;
- }
-
+
public static long getExtractedVal(long val1, long val2, long val3) {
if(val1 == -1 || val2 == -1 || val3 == -1) {
return -1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 674536d..4202fe3 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -32,16 +32,15 @@ import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.Compression;
import org.apache.sysml.lops.Data;
import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.UnaryCP;
-import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
-import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -225,11 +224,7 @@ public abstract class Hop
public void setRequiresCompression(boolean flag) {
_requiresCompression = flag;
}
-
- public boolean requiresCompression() {
- return _requiresCompression;
- }
-
+
public boolean hasMatrixInputWithDifferentBlocksizes()
{
for( Hop c : getInput() ) {
@@ -728,22 +723,6 @@ public abstract class Hop
*/
protected abstract long[] inferOutputCharacteristics( MemoTable memo );
-
-
- /**
- * This function is used only for sanity check.
- * Returns true if estimates for all the hops in the DAG rooted at the current
- * hop are computed. Returns false if any of the hops have INVALID estimate.
- *
- * @return true if estimates computed for all hops rooted at current hop
- */
- public boolean checkEstimates() {
- boolean childStatus = true;
- for (Hop h : this.getInput())
- childStatus = childStatus && h.checkEstimates();
- return childStatus && (_memEstimate != OptimizerUtils.INVALID_SIZE);
- }
-
/**
* Recursively computes memory estimates for all the Hops in the DAG rooted at the
* current hop pointed by <code>this</code>.
@@ -810,17 +789,6 @@ public abstract class Hop
return _input;
}
- /**
- * Create bidirectional links
- *
- * @param h high-level operator
- */
- public void addInput( Hop h )
- {
- _input.add(h);
- h._parent.add(this);
- }
-
public long getRowsInBlock() {
return _rows_in_block;
}
@@ -934,22 +902,6 @@ public abstract class Hop
this.setVisited(VisitStatus.DONE);
}
- public void checkParentChildPointers( )
- throws HopsException
- {
- if( getVisited() == VisitStatus.DONE )
- return;
-
- for( Hop in : getInput() )
- {
- if( !in.getParent().contains(this) )
- throw new HopsException("Parent-Child pointers incorrect.");
- in.checkParentChildPointers();
- }
-
- setVisited(VisitStatus.DONE);
- }
-
public void printMe() throws HopsException {
if (LOG.isDebugEnabled()) {
StringBuilder s = new StringBuilder("");
@@ -1387,11 +1339,7 @@ public abstract class Hop
HopsOpOp2String.put(OpOp2.RBIND, "rbind");
HopsOpOp2String.put(OpOp2.SOLVE, "solve");
}
-
- public static String getOpOp2String( OpOp2 op ) {
- return HopsOpOp2String.get(op);
- }
-
+
protected static final HashMap<Hop.OpOp3, String> HopsOpOp3String;
static {
HopsOpOp3String = new HashMap<Hop.OpOp3, String>();
@@ -1454,33 +1402,6 @@ public abstract class Hop
HopsData2String.put(DataOpTypes.TRANSIENTWRITE, "TWrite");
HopsData2String.put(DataOpTypes.TRANSIENTREAD, "TRead");
}
-
- public static boolean isFunction(OpOp2 op)
- {
- return op == OpOp2.MIN || op == OpOp2.MAX ||
- op == OpOp2.LOG;// || op == OpOp2.CONCAT; //concat is || in Netezza
- }
-
- public static boolean isSupported(OpOp2 op)
- {
- return op != OpOp2.INVALID && op != OpOp2.QUANTILE &&
- op != OpOp2.INTERQUANTILE && op != OpOp2.IQM;
- }
-
- public static boolean isFunction(OpOp1 op)
- {
- return op == OpOp1.SIN || op == OpOp1.TAN || op == OpOp1.COS ||
- op == OpOp1.ABS || op == OpOp1.EXP || op == OpOp1.LOG ||
- op == OpOp1.ROUND || op == OpOp1.SQRT;
- }
-
- public static boolean isBooleanOperation(OpOp2 op)
- {
- return op == OpOp2.AND || op == OpOp2.EQUAL ||
- op == OpOp2.GREATER || op == OpOp2.GREATEREQUAL ||
- op == OpOp2.LESS || op == OpOp2.LESSEQUAL ||
- op == OpOp2.OR;
- }
public static OpOp2 getOpOp2ForOuterVectorOperation(String op)
{
@@ -1505,17 +1426,7 @@ public abstract class Hop
return null;
}
-
- public static ValueType getResultValueType(ValueType vt1, ValueType vt2)
- {
- if(vt1 == ValueType.STRING || vt2 == ValueType.STRING)
- return ValueType.STRING;
- else if(vt1 == ValueType.DOUBLE || vt2 == ValueType.DOUBLE)
- return ValueType.DOUBLE;
- else
- return ValueType.INT;
- }
-
+
/////////////////////////////////////
// methods for dynamic re-compilation
/////////////////////////////////////
@@ -1534,12 +1445,7 @@ public abstract class Hop
{
_requiresRecompile = true;
}
-
- public void unsetRequiresRecompile()
- {
- _requiresRecompile = false;
- }
-
+
/**
* Update the output size information for this hop.
*/
@@ -1778,21 +1684,6 @@ public abstract class Hop
return ret;
}
- public String constructBaseDir()
- {
- StringBuilder sb = new StringBuilder();
- sb.append( ConfigurationManager.getScratchSpace() );
- sb.append( Lop.FILE_SEPARATOR );
- sb.append( Lop.PROCESS_PREFIX );
- sb.append( DMLScript.getUUID() );
- sb.append( Lop.FILE_SEPARATOR );
- sb.append( Lop.FILE_SEPARATOR );
- sb.append( ProgramConverter.CP_ROOT_THREAD_ID );
- sb.append( Lop.FILE_SEPARATOR );
-
- return sb.toString();
- }
-
/**
* Clones the attributes of that and copies it over to this.
*
@@ -1871,11 +1762,7 @@ public abstract class Hop
public String printErrorLocation(){
return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
}
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
+
/**
* Sets the linenumbers of this hop to a given lop.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 2aa421b..a7525a5 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -37,7 +37,6 @@ import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.compile.Dag;
-import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
@@ -51,7 +50,6 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.SparseBlock;
-import org.apache.sysml.runtime.matrix.data.SparseRow;
import org.apache.sysml.runtime.util.IndexRange;
import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer;
@@ -735,37 +733,7 @@ public class OptimizerUtils
{
return estimateSizeExactSparsity(0, 0, 0.0d);
}
-
- /**
- * Estimates the memory footprint of a SparseRow with <code>clen</code>
- * columns and <code>sp</code> sparsity. This method accounts for the
- * overhead incurred by extra cells allocated (but not used) for SparseRow.
- * It assumes that non-zeros are uniformly distributed in the matrix --
- * i.e., #estimated nnz in a given SparseRow = clen*sp.
- *
- * @param clen number of cols
- * @param sp sparsity
- * @return estimated size in bytes
- */
- public static long estimateRowSize(long clen, double sp)
- {
- if ( sp == 0 )
- return 0;
-
- int basicSize = 28;
- int cellSize = 12; // every cell takes 12 (8+4) bytes
- if ( sp == 1 ) {
- return clen * cellSize;
- }
- long numCells = SparseRow.initialCapacity;
- if ( (long) (sp*clen) > numCells ) {
- numCells = (long) (sp*clen);
- }
- long allocatedCells = (long)Math.pow(2, Math.ceil(Math.log(numCells)/Math.log(2)) );
- long rowSize = basicSize + allocatedCells * cellSize;
- return rowSize;
- }
-
+
public static long estimateSizeTextOutput( long rows, long cols, long nnz, OutputInfo oinfo )
{
long bsize = MatrixBlock.estimateSizeOnDisk(rows, cols, nnz);
@@ -898,16 +866,6 @@ public class OptimizerUtils
public static String getUniqueTempFileName() {
return new Dag<Lop>().getNextUniqueFilename();
}
-
- /**
- * Wrapper over internal varname construction for external usage.
- *
- * @param dt data type
- * @return unique variable name
- */
- public static String getUniqueVariableName(DataType dt) {
- return Dag.getNextUniqueVarname(dt);
- }
public static boolean allowsToFilterEmptyBlockOutputs( Hop hop )
throws HopsException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
index a450168..52a966e 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimationWrapper.java
@@ -19,18 +19,12 @@
package org.apache.sysml.hops.cost;
-import java.io.IOException;
-import java.util.ArrayList;
import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-
-import org.apache.sysml.hops.Hop;
-import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.lops.LopsException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.Program;
@@ -98,20 +92,6 @@ public class CostEstimationWrapper
LOG.debug("Finished estimation in "+time.stop()+"ms.");
return costs;
}
-
- public static double getTimeEstimate( ArrayList<Hop> hops, ExecutionContext ec )
- throws DMLRuntimeException, HopsException, LopsException, IOException
- {
- Timing time = new Timing(true);
-
- HashMap<String,VarStats> stats = new HashMap<String, VarStats>();
- LocalVariableMap vars = (ec!=null)? ec.getVariables() : new LocalVariableMap();
-
- double costs = _costEstim.getTimeEstimate(hops, vars, stats);
- LOG.debug("Finished estimation in "+time.stop()+"ms.");
-
- return costs;
- }
private static CostEstimator createCostEstimator( CostType type )
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 65fe83a..955fadb 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -19,21 +19,16 @@
package org.apache.sysml.hops.cost;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.StringTokenizer;
import java.util.Map.Entry;
+import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.hops.Hop;
-import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
-import org.apache.sysml.lops.LopsException;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
@@ -104,24 +99,6 @@ public abstract class CostEstimator
return rGetTimeEstimate(pb, stats, new HashSet<String>(), recursive);
}
- public double getTimeEstimate( ArrayList<Hop> hops, LocalVariableMap vars, HashMap<String,VarStats> stats )
- throws DMLRuntimeException, HopsException, LopsException, IOException
- {
- double costs = 0;
-
- ArrayList<Instruction> linst = Recompiler.recompileHopsDag(null, hops, vars, null, false, 0);
- ProgramBlock pb = new ProgramBlock(null);
- pb.setInstructions(linst);
-
- //obtain stats from symboltable (e.g., during recompile)
- maintainVariableStatistics(vars, stats);
-
- //get cost estimate
- costs = rGetTimeEstimate(pb, stats, new HashSet<String>(), true);
-
- return costs;
- }
-
private double rGetTimeEstimate(ProgramBlock pb, HashMap<String,VarStats> stats, HashSet<String> memoFunc, boolean recursive)
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java b/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
index ba54537..68b5d8e 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/RecompileStatus.java
@@ -19,10 +19,8 @@
package org.apache.sysml.hops.recompile;
-import java.util.ArrayList;
import java.util.HashMap;
-import org.apache.sysml.parser.VariableSet;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
public class RecompileStatus
@@ -39,20 +37,7 @@ public class RecompileStatus
{
return _lastTWrites;
}
-
- public void clearStatus()
- {
- _lastTWrites.clear();
- }
-
- public void clearStatus(VariableSet vars)
- {
- ArrayList<String> lvars = new ArrayList<String>(vars.getVariableNames());
- for( String var : lvars ) {
- _lastTWrites.remove(var);
- }
- }
-
+
@Override
public Object clone()
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 327b097..8b121d7 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -1299,12 +1299,7 @@ public class Recompiler
for( Hop hop : hops ) //for all hop roots
extractDAGOutputStatistics(hop, vars, overwrite);
}
-
- public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars)
- {
- extractDAGOutputStatistics(hop, vars, true);
- }
-
+
public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars, boolean overwrite)
{
if( hop instanceof DataOp && ((DataOp)hop).getDataOpType()==DataOpTypes.TRANSIENTWRITE ) //for all writes to symbol table
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index 7d0edd5..8ccb531 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -234,14 +234,7 @@ public class HopRewriteUtils
parent.getInput().remove( posChild );
child.getParent().remove( parent );
}
-
- public static void removeChildReferenceByPos( Hop parent, Hop child, int posChild, int posParent )
- {
- //remove child reference
- parent.getInput().remove( posChild );
- child.getParent().remove( posParent );
- }
-
+
public static void removeAllChildReferences( Hop parent )
{
//remove parent reference from all childs
@@ -799,23 +792,7 @@ public class HopRewriteUtils
return ret;
}
-
- public static double getBasic1NSequenceMax(Hop hop)
- throws HopsException
- {
- if( hop instanceof DataGenOp )
- {
- DataGenOp dgop = (DataGenOp) hop;
- if( dgop.getOp() == DataGenMethod.SEQ ){
- Hop to = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_TO));
- if( to instanceof LiteralOp )
- return getDoubleValueSafe((LiteralOp)to);
- }
- }
-
- throw new HopsException("Failed to retrieve 'to' argument from basic 1-N sequence.");
- }
-
+
public static LiteralOp getBasic1NSequenceMaxLiteral(Hop hop)
throws HopsException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java
index b5b8b80..a5ed9e9 100644
--- a/src/main/java/org/apache/sysml/lops/Lop.java
+++ b/src/main/java/org/apache/sysml/lops/Lop.java
@@ -627,10 +627,6 @@ public abstract class Lop
public String printErrorLocation(){
return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
}
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
//TODO: Leo This might get confused with Rand.getInstructions
public String getInstructions(String input, String rowl, String rowu,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/lops/PartialAggregate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/PartialAggregate.java b/src/main/java/org/apache/sysml/lops/PartialAggregate.java
index 9ec23f4..ce54655 100644
--- a/src/main/java/org/apache/sysml/lops/PartialAggregate.java
+++ b/src/main/java/org/apache/sysml/lops/PartialAggregate.java
@@ -135,12 +135,7 @@ public class PartialAggregate extends Lop
{
_dropCorr = true;
}
-
- public static CorrectionLocationType decodeCorrectionLocation(String loc)
- {
- return CorrectionLocationType.valueOf(loc);
- }
-
+
/**
* This method computes the location of "correction" terms in the output
* produced by PartialAgg instruction.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/lops/Ternary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Ternary.java b/src/main/java/org/apache/sysml/lops/Ternary.java
index 7a79088..a2d674b 100644
--- a/src/main/java/org/apache/sysml/lops/Ternary.java
+++ b/src/main/java/org/apache/sysml/lops/Ternary.java
@@ -331,21 +331,6 @@ public class Ternary extends Lop
return sb.toString();
}
-
- public static String getOpcode(OperationTypes type)
- {
- switch( type )
- {
- case CTABLE_TRANSFORM: return "ctabletransform";
- case CTABLE_TRANSFORM_SCALAR_WEIGHT: return "ctabletransformscalarweight";
- case CTABLE_EXPAND_SCALAR_WEIGHT: return "ctableexpandscalarweight";
- case CTABLE_TRANSFORM_HISTOGRAM: return "ctabletransformhistogram";
- case CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM: return "ctabletransformweightedhistogram";
-
- default:
- throw new UnsupportedOperationException("Ternary operation code is not defined: " + type);
- }
- }
public static OperationTypes getOperationType(String opcode)
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/lops/Transform.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Transform.java b/src/main/java/org/apache/sysml/lops/Transform.java
index e0c2f35..d677f6b 100644
--- a/src/main/java/org/apache/sysml/lops/Transform.java
+++ b/src/main/java/org/apache/sysml/lops/Transform.java
@@ -272,30 +272,4 @@ public class Transform extends Lop
return sb.toString();
}
-
- public static Transform constructTransformLop(Lop input1, OperationTypes op, DataType dt, ValueType vt) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.Transform ) {
- return (Transform)lop;
- }
- }
- Transform retVal = new Transform(input1, op, dt, vt);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
- public static Transform constructTransformLop(Lop input1, OperationTypes op, DataType dt, ValueType vt, ExecType et) {
-
- for (Lop lop : input1.getOutputs()) {
- if ( lop.type == Lop.Type.Transform ) {
- return (Transform)lop;
- }
- }
- Transform retVal = new Transform(input1, op, dt, vt, et);
- retVal.setAllPositions(input1.getBeginLine(), input1.getBeginColumn(), input1.getEndLine(), input1.getEndColumn());
- return retVal;
- }
-
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/Dag.java b/src/main/java/org/apache/sysml/lops/compile/Dag.java
index 5a15c9f..8a2fa1c 100644
--- a/src/main/java/org/apache/sysml/lops/compile/Dag.java
+++ b/src/main/java/org/apache/sysml/lops/compile/Dag.java
@@ -230,13 +230,7 @@ public class Dag<N extends Lop>
nodes.add(node);
return true;
}
-
- public ArrayList<Instruction> getJobs(DMLConfig config)
- throws LopsException, IOException, DMLRuntimeException
- {
- return getJobs(null, config);
- }
-
+
/**
* Method to compile a dag generically
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/lops/compile/JobType.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/JobType.java b/src/main/java/org/apache/sysml/lops/compile/JobType.java
index 91d3f1d..8403497 100644
--- a/src/main/java/org/apache/sysml/lops/compile/JobType.java
+++ b/src/main/java/org/apache/sysml/lops/compile/JobType.java
@@ -232,22 +232,6 @@ public enum JobType
return (int) Math.pow(2, id-1);
}
- public JobType findJobTypeById(int id) {
- for (JobType jt : JobType.values()) {
- if (jt.getId() == id)
- return jt;
- }
- return null;
- }
-
- public JobType findJobTypeByName(String name) {
- for (JobType jt : JobType.values()) {
- if (jt.getName().equalsIgnoreCase(name))
- return jt;
- }
- return null;
- }
-
public static int getNumJobTypes() {
return values().length;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
index 2e64674..25f1fa9 100644
--- a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
+++ b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
@@ -67,28 +67,11 @@ public class ConditionalPredicate
public void setBeginColumn(int passed) { _beginColumn = passed; }
public void setEndLine(int passed) { _endLine = passed; }
public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(String fname, int blp, int bcp, int elp, int ecp){
- _filename = fname;
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
public String getFilename() { return _filename; }
public int getBeginLine() { return _beginLine; }
public int getBeginColumn() { return _beginColumn; }
public int getEndLine() { return _endLine; }
public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/DMLProgram.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLProgram.java b/src/main/java/org/apache/sysml/parser/DMLProgram.java
index d6a0deb..292a13c 100644
--- a/src/main/java/org/apache/sysml/parser/DMLProgram.java
+++ b/src/main/java/org/apache/sysml/parser/DMLProgram.java
@@ -67,11 +67,7 @@ public class DMLProgram
public HashMap<String,DMLProgram> getNamespaces(){
return _namespaces;
}
-
- public void addStatementBlock(StatementBlock b, int pos) {
- _blocks.add(pos,b) ;
- }
-
+
public void addStatementBlock(StatementBlock b){
_blocks.add(b);
}
@@ -143,10 +139,6 @@ public class DMLProgram
public StatementBlock getStatementBlock(int i){
return _blocks.get(i);
}
-
- public void setStatementBlock(int i, StatementBlock sb) {
- _blocks.set(i, sb);
- }
public void mergeStatementBlocks(){
_blocks = StatementBlock.mergeStatementBlocks(_blocks);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index e7e37c8..545fe12 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -2962,15 +2962,6 @@ public class DMLTranslator
h.setColsInBlock(id.getColumnsInBlock());
}
- public void setIdentifierParams(Hop h, Hop source) {
-
- h.setDim1(source.getDim1());
- h.setDim2(source.getDim2());
- h.setNnz(source.getNnz());
- h.setRowsInBlock(source.getRowsInBlock());
- h.setColsInBlock(source.getColsInBlock());
- }
-
private boolean prepareReadAfterWrite( DMLProgram prog, HashMap<String, DataIdentifier> pWrites )
throws LanguageException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/DataIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataIdentifier.java b/src/main/java/org/apache/sysml/parser/DataIdentifier.java
index 41dffe8..158829a 100644
--- a/src/main/java/org/apache/sysml/parser/DataIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/DataIdentifier.java
@@ -71,40 +71,7 @@ public class DataIdentifier extends Identifier
_kind = null;
_defaultValue = null;
}
-
- public void setTypeInfo( String valueType, String dataType) throws ParseException{
-
- if (valueType.equalsIgnoreCase("int") || valueType.equalsIgnoreCase("integer"))
- this.setValueType(ValueType.INT);
- else if (valueType.equalsIgnoreCase("double"))
- this.setValueType(ValueType.DOUBLE);
- else if (valueType.equalsIgnoreCase("string"))
- this.setValueType(ValueType.STRING);
- else if (valueType.equalsIgnoreCase("boolean"))
- this.setValueType(ValueType.BOOLEAN);
- else if (valueType.equalsIgnoreCase("object"))
- this.setValueType(ValueType.OBJECT);
- else {
- // provide location for this exception in the parser
- LOG.error(this.printErrorLocation() + "function parameter has unknown value type " + valueType);
- throw new ParseException(this.printErrorLocation() + "function parameter has unknown value type " + valueType);
- }
-
- if (dataType.equalsIgnoreCase("object"))
- this.setDataType(DataType.OBJECT);
- else if (dataType.equalsIgnoreCase("SCALAR"))
- this.setDataType(DataType.SCALAR);
- else if (dataType.equalsIgnoreCase("MATRIX"))
- this.setDataType(DataType.MATRIX);
- else {
- // provide location for this exception in the parser
- LOG.error(this.printErrorLocation() + "function parameter has unknown data type " + valueType);
- throw new ParseException(this.printErrorLocation() + "function parameter has unknown data type " + valueType);
- }
-
- }
-
public String getName(){
return _name;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/FunctionStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/FunctionStatement.java b/src/main/java/org/apache/sysml/parser/FunctionStatement.java
index a1c46d9..ea2ed5c 100644
--- a/src/main/java/org/apache/sysml/parser/FunctionStatement.java
+++ b/src/main/java/org/apache/sysml/parser/FunctionStatement.java
@@ -21,8 +21,6 @@ package org.apache.sysml.parser;
import java.util.ArrayList;
-import org.apache.sysml.lops.Lop;
-
public class FunctionStatement extends Statement
{
@@ -67,11 +65,7 @@ public class FunctionStatement extends Statement
public String getName(){
return _name;
}
-
- public void addStatementBlock(StatementBlock sb){
- _body.add(sb);
- }
-
+
public ArrayList<StatementBlock> getBody(){
return _body;
}
@@ -139,18 +133,4 @@ public class FunctionStatement extends Statement
LOG.warn(this.printWarningLocation() + " -- should not call variablesRead from FunctionStatement ");
return new VariableSet();
}
-
- public static String[] createFunctionCallVariables( ArrayList<Lop> lops )
- {
- String[] ret = new String[lops.size()]; //vars in order
-
- for( int i=0; i<lops.size(); i++ )
- {
- Lop llops = lops.get(i);
- if( llops.getType()==Lop.Type.Data )
- ret[i] = llops.getOutputParameters().getLabel();
- }
-
- return ret;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/IfStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IfStatement.java b/src/main/java/org/apache/sysml/parser/IfStatement.java
index b3d999a..0eea248 100644
--- a/src/main/java/org/apache/sysml/parser/IfStatement.java
+++ b/src/main/java/org/apache/sysml/parser/IfStatement.java
@@ -100,32 +100,7 @@ public class IfStatement extends Statement
if (!_elseBody.isEmpty())
_elseBody = StatementBlock.mergeStatementBlocks(_elseBody);
}
-
- public VariableSet variablesReadIfBody() {
-
- return null;
-
- }
-
- public VariableSet variablesReadElseBody() {
-
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesReadElseBody from IfStatement ");
- return null;
- }
-
- public VariableSet variablesUpdatedIfBody() {
-
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesUpdatedIfBody from IfStatement ");
- return null;
- }
-
- public VariableSet variablesUpdatedElseBody() {
-
- LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesUpdatedElseBody from IfStatement ");
- return null;
- }
-
-
+
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append("if ( ");
@@ -145,11 +120,6 @@ public class IfStatement extends Statement
return sb.toString();
}
-
- public VariableSet variablesKill() {
- return new VariableSet();
- }
-
@Override
public VariableSet variablesRead() {
LOG.warn("WARNING: line " + this.getBeginLine() + ", column " + this.getBeginColumn() + " -- should not call variablesRead from IfStatement ");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
index 11110b5..637e445 100644
--- a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
@@ -709,35 +709,6 @@ public class IndexedIdentifier extends DataIdentifier
return result;
}
- public void setAllPositions (String filename, int blp, int bcp, ArrayList<ArrayList<Expression>> exprListList){
- setFilename(filename);
- setBeginLine(blp);
- setBeginColumn(bcp);
-
- Expression rightMostIndexExpr = null;
- if (exprListList != null){
- for(ArrayList<Expression> exprList : exprListList){
- if (exprList != null){
- for (Expression expr : exprList){
- if (expr != null) {
- rightMostIndexExpr = expr;
- }
- }
- }
- }
- }
-
- if (rightMostIndexExpr != null){
- setEndLine(rightMostIndexExpr.getEndLine());
- setEndColumn(rightMostIndexExpr.getEndColumn());
- }
- else {
- setEndLine(blp);
- setEndColumn(bcp);
- }
- }
-
-
public void setProperties(Identifier i){
_dataType = i.getDataType();
_valueType = i.getValueType();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java b/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
index 9cadc3d..9133e33 100644
--- a/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
+++ b/src/main/java/org/apache/sysml/parser/LiveVariableAnalysis.java
@@ -79,11 +79,6 @@ public abstract class LiveVariableAnalysis
public abstract VariableSet initializebackwardLV(VariableSet loPassed) throws LanguageException;
public abstract VariableSet analyze(VariableSet loPassed) throws LanguageException;
-
- public void updateLiveVariablesIn(VariableSet liveIn){
- updateLiveVariables(_liveIn,liveIn);
- }
-
public void updateLiveVariablesOut(VariableSet liveOut){
updateLiveVariables(_liveOut,liveOut);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/OutputStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/OutputStatement.java b/src/main/java/org/apache/sysml/parser/OutputStatement.java
index 3d1b7bb..ec58d57 100644
--- a/src/main/java/org/apache/sysml/parser/OutputStatement.java
+++ b/src/main/java/org/apache/sysml/parser/OutputStatement.java
@@ -56,10 +56,6 @@ public class OutputStatement extends Statement
filename, blp, bcp, elp, ecp);
}
- public void setExprParam(String name, Expression value) {
- _paramsExpr.addVarParam(name, value);
- }
-
public static boolean isValidParamName(String key){
for (String paramName : WRITE_VALID_PARAM_NAMES)
if (paramName.equals(key))
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
index d632249..c135b74 100644
--- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
@@ -168,11 +168,7 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
public void addVarParam(String name, Expression value){
_varParams.put(name, value);
}
-
- public void removeVarParam(String name) {
- _varParams.remove(name);
- }
-
+
/**
* Validate parse tree : Process BuiltinFunction Expression in an assignment
* statement
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/Statement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Statement.java b/src/main/java/org/apache/sysml/parser/Statement.java
index 8268796..c2e00ea 100644
--- a/src/main/java/org/apache/sysml/parser/Statement.java
+++ b/src/main/java/org/apache/sysml/parser/Statement.java
@@ -115,26 +115,6 @@ public abstract class Statement
public String printWarningLocation(){
return "WARNING: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
}
-
- public String printInfoLocation(){
- return "INFO: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printErrorLocation(int beginLine, int beginColumn){
- return "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
- }
-
- public String printWarningLocation(int beginLine, int beginColumn){
- return "WARNING: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
- }
-
- public String printInfoLocation(int beginLine, int beginColumn){
- return "INFO: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
- }
-
- public void raiseValidateError( String msg ) throws LanguageException {
- raiseValidateError(msg, false, null);
- }
public void raiseValidateError( String msg, boolean conditional ) throws LanguageException {
raiseValidateError(msg, conditional, null);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/StatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java b/src/main/java/org/apache/sysml/parser/StatementBlock.java
index af085d5..73fdd94 100644
--- a/src/main/java/org/apache/sysml/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java
@@ -91,21 +91,7 @@ public class StatementBlock extends LiveVariableAnalysis
this._endColumn = s.getEndColumn();
}
-
- public void replaceStatement(int index, Statement passedStmt){
- this._statements.set(index, passedStmt);
-
- if (index == 0){
- this._beginLine = passedStmt.getBeginLine();
- this._beginColumn = passedStmt.getBeginColumn();
- }
-
- else if (index == this._statements.size() -1){
- this._endLine = passedStmt.getEndLine();
- this._endColumn = passedStmt.getEndColumn();
- }
- }
-
+
public void addStatementBlock(StatementBlock s){
for (int i = 0; i < s.getNumStatements(); i++){
_statements.add(s.getStatement(i));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/VariableSet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/VariableSet.java b/src/main/java/org/apache/sysml/parser/VariableSet.java
index 819e9e3..b1f5067 100644
--- a/src/main/java/org/apache/sysml/parser/VariableSet.java
+++ b/src/main/java/org/apache/sysml/parser/VariableSet.java
@@ -62,12 +62,7 @@ public class VariableSet
_variables.remove(var);
}
}
-
- public void removeVariable(String name)
- {
- _variables.remove(name);
- }
-
+
public boolean containsVariable(String name){
return _variables.containsKey(name);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
index bcdeffd..43eabcb 100644
--- a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
@@ -86,10 +86,6 @@ public abstract class CommonSyntacticValidator {
functions = (null != prepFunctions) ? prepFunctions : new HashSet<String>();
}
- protected void notifyErrorListeners(String message, int line, int charPositionInLine) {
- errorListener.validationError(line, charPositionInLine, message);
- }
-
protected void notifyErrorListeners(String message, Token op) {
errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
}
@@ -156,19 +152,6 @@ public abstract class CommonSyntacticValidator {
notifyErrorListeners("Namespace Conflict: '" + namespace + "' already defined as " + sources.get(namespace), ctx.start);
}
}
-
- protected boolean validateBuiltinFunctions(String function) {
- String functionName = function.replaceAll(" ", "").trim();
- if(functionName.equals("write") || functionName.equals(DMLProgram.DEFAULT_NAMESPACE + namespaceResolutionOp() + "write")) {
- return validateBuiltinWriteFunction(function);
- }
- return true;
- }
-
- protected boolean validateBuiltinWriteFunction(String function) {
- return true;
- }
-
protected void setFileLineColumn(Expression expr, ParserRuleContext ctx) {
String txt = ctx.getText();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java b/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
index fa3f9d9..90d0265 100644
--- a/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
+++ b/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
@@ -317,14 +317,6 @@ public class CustomErrorListener extends BaseErrorListener {
}
/**
- * Clear the list of parse issues.
- *
- */
- public void clearParseIssues() {
- parseIssues.clear();
- }
-
- /**
* Generate a message displaying information about the parse issues that
* occurred.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/compress/ColGroupBitmap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupBitmap.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupBitmap.java
index a9fdc0f..dac18ef 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupBitmap.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupBitmap.java
@@ -25,11 +25,10 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
-import java.util.TreeMap;
import java.util.Map.Entry;
+import java.util.TreeMap;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.compress.utils.LinearAlgebraUtils;
import org.apache.sysml.runtime.functionobjects.Builtin;
import org.apache.sysml.runtime.functionobjects.Builtin.BuiltinCode;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -325,17 +324,6 @@ public abstract class ColGroupBitmap extends ColGroup
return val;
}
- protected final void sumAllValues(double[] b, double[] c)
- {
- final int numVals = getNumValues();
- final int numCols = getNumCols();
-
- //vectMultiplyAdd over cols instead of dotProduct over vals because
- //usually more values than columns
- for( int i=0, off=0; i<numCols; i++, off+=numVals )
- LinearAlgebraUtils.vectMultiplyAdd(b[i], _values, c, off, 0, numVals);
- }
-
protected final double mxxValues(int bitmapIx, Builtin builtin)
{
final int numCols = getNumCols();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/compress/PlanningBinPacker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/PlanningBinPacker.java b/src/main/java/org/apache/sysml/runtime/compress/PlanningBinPacker.java
index 7c3d8d2..70308bb 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/PlanningBinPacker.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/PlanningBinPacker.java
@@ -20,14 +20,10 @@
package org.apache.sysml.runtime.compress;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import org.apache.commons.math3.random.RandomDataGenerator;
-
/**
* Used for the finding columns to co-code
*
@@ -53,26 +49,6 @@ public class PlanningBinPacker
return packFirstFit(_items, _itemWeights);
}
- /**
- * shuffling the items to make some potential for having bins of different
- * sizes when consecutive columns are of close cardinalities
- *
- * @return key: available space, value: list of the bins that have that free
- * space
- */
- public TreeMap<Float, List<List<Integer>>> packFirstFitShuffled() {
- RandomDataGenerator rnd = new RandomDataGenerator();
- int[] permutation = rnd.nextPermutation(_items.size(), _items.size());
- List<Integer> shuffledItems = new ArrayList<Integer>(_items.size());
- List<Float> shuffledWeights = new ArrayList<Float>(_items.size());
- for (int ix : permutation) {
- shuffledItems.add(_items.get(ix));
- shuffledWeights.add(_itemWeights.get(ix));
- }
-
- return packFirstFit(shuffledItems, shuffledWeights);
- }
-
private TreeMap<Float, List<List<Integer>>> packFirstFit(List<Integer> items, List<Float> itemWeights)
{
// when searching for a bin, the first bin in the list is used
@@ -114,41 +90,6 @@ public class PlanningBinPacker
return bins;
}
- /**
- * NOTE: upper bound is 11/9 OPT + 6/9 (~1.22 OPT)
- *
- * @return sorted map of ?
- */
- public TreeMap<Float, List<List<Integer>>> packFirstFitDescending() {
- // sort items descending based on their weights
- Integer[] indexes = new Integer[_items.size()];
- for (int i = 0; i < indexes.length; i++)
- indexes[i] = i;
- Arrays.sort(indexes, new Comparator<Integer>() {
-
- @Override
- public int compare(Integer o1, Integer o2) {
- return _itemWeights.get(o1).compareTo(_itemWeights.get(o2));
- }
- });
- List<Integer> sortedItems = new ArrayList<Integer>();
- List<Float> sortedItemWeights = new ArrayList<Float>();
- for (int i = indexes.length - 1; i >= 0; i--) {
- sortedItems.add(_items.get(i));
- sortedItemWeights.add(_itemWeights.get(i));
- }
- return packFirstFit(sortedItems, sortedItemWeights);
- }
-
- /**
- * NOTE: upper bound is 71/60 OPT + 6/9 (~1.18 OPT)
- *
- * @return sorted map of ?
- */
- public TreeMap<Float, List<List<Integer>>> packModifiedFirstFitDescending() {
- throw new UnsupportedOperationException("Not implemented yet!");
- }
-
private List<List<Integer>> createBinList() {
List<List<Integer>> binList = new ArrayList<List<Integer>>();
binList.add(new ArrayList<Integer>());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java b/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java
index cc5932b..eb0040f 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java
@@ -70,10 +70,6 @@ public class CompressedSizeEstimatorSample extends CompressedSizeEstimator
_sampleRows = sampleRows;
}
- public void resampleRows(int sampleSize) {
- _sampleRows = getSortedUniformSample(_numRows, sampleSize);
- }
-
@Override
public CompressedSizeInfo estimateCompressedColGroupSize(int[] colIndexes)
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/compress/utils/ConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/utils/ConverterUtils.java b/src/main/java/org/apache/sysml/runtime/compress/utils/ConverterUtils.java
index f4d9f1c..37b2984 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/utils/ConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/utils/ConverterUtils.java
@@ -19,7 +19,6 @@
package org.apache.sysml.runtime.compress.utils;
-import java.util.ArrayList;
import java.util.Arrays;
import org.apache.sysml.runtime.compress.ColGroup;
@@ -71,19 +70,4 @@ public class ConverterUtils
else
return vector.getDenseBlock();
}
-
- public static MatrixBlock getUncompressedColBlock( ColGroup group )
- {
- MatrixBlock ret = null;
- if( group instanceof ColGroupUncompressed ) {
- ret = ((ColGroupUncompressed) group).getData();
- }
- else {
- ArrayList<ColGroup> tmpGroup = new ArrayList<ColGroup>(Arrays.asList(group));
- ColGroupUncompressed decompressedCols = new ColGroupUncompressed(tmpGroup);
- ret = decompressedCols.getData();
- }
-
- return ret;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/compress/utils/LinearAlgebraUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/utils/LinearAlgebraUtils.java b/src/main/java/org/apache/sysml/runtime/compress/utils/LinearAlgebraUtils.java
index 7a4a013..3bf0ad4 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/utils/LinearAlgebraUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/utils/LinearAlgebraUtils.java
@@ -28,86 +28,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
*/
public class LinearAlgebraUtils {
- public static double dotProduct(double[] a, double[] b, final int len)
- {
- double val = 0;
- final int bn = len % 8;
-
- // compute rest
- for (int i = 0; i < bn; i++)
- val += a[i] * b[i];
-
- // unrolled 8-block (for better instruction-level parallelism)
- for (int i = bn; i < len; i += 8) {
- // read 64B cachelines of a and b
- // compute cval' = sum(a * b) + cval
- val += a[i + 0] * b[i + 0]
- + a[i + 1] * b[i + 1]
- + a[i + 2] * b[i + 2]
- + a[i + 3] * b[i + 3]
- + a[i + 4] * b[i + 4]
- + a[i + 5] * b[i + 5]
- + a[i + 6] * b[i + 6]
- + a[i + 7] * b[i + 7];
- }
-
- // scalar result
- return val;
- }
-
- public static double dotProduct( double[] a, double[] b, int ai, int bi, final int len )
- {
- double val = 0;
- final int bn = len%8;
-
- //compute rest
- for( int i = 0; i < bn; i++, ai++, bi++ )
- val += a[ ai ] * b[ bi ];
-
- //unrolled 8-block (for better instruction-level parallelism)
- for( int i = bn; i < len; i+=8, ai+=8, bi+=8 )
- {
- //read 64B cachelines of a and b
- //compute cval' = sum(a * b) + cval
- val += a[ ai+0 ] * b[ bi+0 ]
- + a[ ai+1 ] * b[ bi+1 ]
- + a[ ai+2 ] * b[ bi+2 ]
- + a[ ai+3 ] * b[ bi+3 ]
- + a[ ai+4 ] * b[ bi+4 ]
- + a[ ai+5 ] * b[ bi+5 ]
- + a[ ai+6 ] * b[ bi+6 ]
- + a[ ai+7 ] * b[ bi+7 ];
- }
-
- //scalar result
- return val;
- }
-
- public static void vectAdd( double[] a, double[] c, int ai, int ci, final int len )
- {
- final int bn = len%8;
-
- //rest, not aligned to 8-blocks
- for( int j = 0; j < bn; j++, ai++, ci++)
- c[ ci ] += a[ ai ];
-
- //unrolled 8-block (for better instruction-level parallelism)
- for( int j = bn; j < len; j+=8, ai+=8, ci+=8)
- {
- //read 64B cachelines of a and c
- //compute c' = c * a
- //write back 64B cacheline of c = c'
- c[ ci+0 ] += a[ ai+0 ];
- c[ ci+1 ] += a[ ai+1 ];
- c[ ci+2 ] += a[ ai+2 ];
- c[ ci+3 ] += a[ ai+3 ];
- c[ ci+4 ] += a[ ai+4 ];
- c[ ci+5 ] += a[ ai+5 ];
- c[ ci+6 ] += a[ ai+6 ];
- c[ ci+7 ] += a[ ai+7 ];
- }
- }
-
public static void vectAdd( final double aval, double[] c, char[] bix, final int bi, final int ci, final int len )
{
final int bn = len%8;
@@ -152,53 +72,6 @@ public class LinearAlgebraUtils {
}
}
- public static void vectMultiplyAdd( final double aval, double[] b, double[] c, int[] bix, final int bi, final int ci, final int len )
- {
- final int bn = (len-bi)%8;
-
- //rest, not aligned to 8-blocks
- for( int j = bi; j < bi+bn; j++ )
- c[ ci + bix[j] ] += aval * b[ j ];
-
- //unrolled 8-block (for better instruction-level parallelism)
- for( int j = bi+bn; j < len; j+=8 )
- {
- c[ ci+bix[j+0] ] += aval * b[ j+0 ];
- c[ ci+bix[j+1] ] += aval * b[ j+1 ];
- c[ ci+bix[j+2] ] += aval * b[ j+2 ];
- c[ ci+bix[j+3] ] += aval * b[ j+3 ];
- c[ ci+bix[j+4] ] += aval * b[ j+4 ];
- c[ ci+bix[j+5] ] += aval * b[ j+5 ];
- c[ ci+bix[j+6] ] += aval * b[ j+6 ];
- c[ ci+bix[j+7] ] += aval * b[ j+7 ];
- }
- }
-
- public static void vectMultiplyAdd( final double aval, double[] b, double[] c, int bi, int ci, final int len )
- {
- final int bn = len%8;
-
- //rest, not aligned to 8-blocks
- for( int j = 0; j < bn; j++, bi++, ci++)
- c[ ci ] += aval * b[ bi ];
-
- //unrolled 8-block (for better instruction-level parallelism)
- for( int j = bn; j < len; j+=8, bi+=8, ci+=8)
- {
- //read 64B cachelines of b and c
- //compute c' = aval * b + c
- //write back 64B cacheline of c = c'
- c[ ci+0 ] += aval * b[ bi+0 ];
- c[ ci+1 ] += aval * b[ bi+1 ];
- c[ ci+2 ] += aval * b[ bi+2 ];
- c[ ci+3 ] += aval * b[ bi+3 ];
- c[ ci+4 ] += aval * b[ bi+4 ];
- c[ ci+5 ] += aval * b[ bi+5 ];
- c[ ci+6 ] += aval * b[ bi+6 ];
- c[ ci+7 ] += aval * b[ bi+7 ];
- }
- }
-
public static double vectSum( double[] a, char[] bix, final int ai, final int bi, final int len )
{
double val = 0;
@@ -249,18 +122,6 @@ public class LinearAlgebraUtils {
return val;
}
- public static void copyUpperToLowerTriangle( MatrixBlock ret )
- {
- double[] c = ret.getDenseBlock();
- final int m = ret.getNumRows();
- final int n = ret.getNumColumns();
-
- //copy symmetric values
- for( int i=0, uix=0; i<m; i++, uix+=n )
- for( int j=i+1, lix=j*n+i; j<n; j++, lix+=n )
- c[ lix ] = c[ uix+j ];
- }
-
public static void copyNonZerosToRowCol( MatrixBlock ret, MatrixBlock tmp, int ix )
{
for(int i=0; i<tmp.getNumColumns(); i++) {
@@ -271,29 +132,4 @@ public class LinearAlgebraUtils {
}
}
}
-
- /**
- * Obtain the index of the closest element in a to the value x.
- *
- * @param a array of ints
- * @param x value
- * @return the index of the closest element in a to the value x
- */
- public static int getClosestK(int[] a, int x) {
-
- int low = 0;
- int high = a.length - 1;
-
- while (low < high) {
- int mid = (low + high) / 2;
- int d1 = Math.abs(a[mid] - x);
- int d2 = Math.abs(a[mid + 1] - x);
- if (d2 <= d1) {
- low = mid + 1;
- } else {
- high = mid;
- }
- }
- return high;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index f01c049..3c72ca9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -909,13 +909,6 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
throw new RuntimeException("Should never come here");
}
- public void printMe() {
- //System.out.println("***** INSTRUCTION BLOCK *****");
- for (Instruction i : this._inst) {
- i.printMe();
- }
- }
-
public HashMap<String,String> getOtherParams() {
return _otherParams;
}
@@ -935,24 +928,6 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
private Collection<String> _skipInReblock = new HashSet<String>();
private Collection<String> _skipOutReblock = new HashSet<String>();
- public void setSkippedReblockLists( Collection<String> varsIn, Collection<String> varsOut )
- {
- _skipInReblock.clear();
- _skipOutReblock.clear();
-
- if( varsIn!=null || varsOut!=null )
- {
- if( varsIn != null )
- _skipInReblock.addAll(varsIn);
- if( varsOut != null )
- _skipOutReblock.addAll(varsOut);
-
- //regenerate instructions
- createInstructions();
- }
- }
-
-
@Override
public ArrayList<Instruction> getInstructions()
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
index be7bada..103ce70 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
@@ -81,11 +81,7 @@ public class ForProgramBlock extends ProgramBlock
public void setIncrementInstructions(ArrayList<Instruction> instructions) {
_incrementInstructions = instructions;
}
-
- public void addExitInstruction(Instruction inst) {
- _exitInstructions.add(inst);
- }
-
+
public ArrayList<Instruction> getExitInstructions() {
return _exitInstructions;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
index 136850c..63757ac 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
@@ -19,17 +19,15 @@
package org.apache.sysml.runtime.controlprogram;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.StringTokenizer;
+
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.instructions.cp.Data;
-import org.apache.sysml.runtime.instructions.spark.data.LineageObject;
-
-import java.util.HashMap;
-import java.util.StringTokenizer;
-import java.util.Map.Entry;
-import java.util.Set;
/**
* Replaces <code>HashMap⟨String, Data⟩</code> as the table of
@@ -85,13 +83,6 @@ public class LocalVariableMap implements Cloneable
localMap.put( name, val );
}
- public void putAll( LocalVariableMap vars )
- {
- if( vars == this || vars == null )
- return;
- localMap.putAll (vars.localMap);
- }
-
public Data remove( String name )
{
return localMap.remove( name );
@@ -107,31 +98,6 @@ public class LocalVariableMap implements Cloneable
return localMap.containsValue(d);
}
- public boolean hasReferences( LineageObject bo )
- {
- for( Data tmpdat : localMap.values() )
- if ( tmpdat instanceof MatrixObject ) {
- MatrixObject mo = (MatrixObject)tmpdat;
- if( mo.getBroadcastHandle()==bo || mo.getRDDHandle()==bo )
- return true;
- }
- return false;
- }
-
- public int getNumReferences( Data d, boolean earlyAbort )
- {
- if ( d == null )
- return 0;
-
- int refCount = 0;
- for( Data tmpdat : localMap.values() )
- if ( tmpdat == d )
- if( ++refCount > 1 && earlyAbort )
- return refCount;
-
- return refCount;
- }
-
public String serialize()
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index e1986c8..eb504ca 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -432,16 +432,8 @@ public class ProgramBlock
public int getBeginColumn() { return _beginColumn; }
public int getEndLine() { return _endLine; }
public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
+
public String printBlockErrorLocation(){
return "ERROR: Runtime error in program block generated from statement block between lines " + _beginLine + " and " + _endLine + " -- ";
}
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
index b77b9ea..c435884 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
@@ -69,11 +69,7 @@ public class WhileProgramBlock extends ProgramBlock
public void setExitInstructions1(ArrayList<Instruction> predicate) {
_predicate = predicate;
}
-
- public void addExitInstruction(Instruction inst) {
- _exitInstructions.add(inst);
- }
-
+
public ArrayList<Instruction> getPredicate() {
return _predicate;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
index a03f9d3..f40ae62 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheStatistics.java
@@ -51,7 +51,6 @@ public class CacheStatistics
}
//hit statistics (for acquire read)
- private static AtomicLong _numHitsTotal = null;
private static AtomicLong _numHitsMem = null;
private static AtomicLong _numHitsFSBuff = null;
private static AtomicLong _numHitsFS = null;
@@ -75,7 +74,6 @@ public class CacheStatistics
public static void reset()
{
- _numHitsTotal = new AtomicLong(0);
_numHitsMem = new AtomicLong(0);
_numHitsFSBuff = new AtomicLong(0);
_numHitsFS = new AtomicLong(0);
@@ -90,22 +88,7 @@ public class CacheStatistics
_ctimeRelease = new AtomicLong(0);
_ctimeExport = new AtomicLong(0);
}
-
- public static void incrementTotalHits()
- {
- _numHitsTotal.incrementAndGet();
- }
-
- public static void incrementTotalHits(int delta)
- {
- _numHitsTotal.addAndGet(delta);
- }
-
- public static long getTotalHits()
- {
- return _numHitsTotal.get();
- }
-
+
public static void incrementMemHits()
{
_numHitsMem.incrementAndGet();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 251b04f..325fa03 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -138,13 +138,7 @@ public class ExecutionContext
{
return _variables.get(varname).getMetaData();
}
-
- public void removeMetaData(String varname)
- throws DMLRuntimeException
- {
- _variables.get(varname).removeMetaData();
- }
-
+
public MatrixObject getMatrixObject(String varname)
throws DMLRuntimeException
{
@@ -243,24 +237,6 @@ public class ExecutionContext
}
/**
- * Allocates a sparse matrix in CSR format on the GPU.
- * Assumes that mat.getNumRows() returns a valid number
- *
- * @param varName variable name
- * @param nnz number of non zeroes
- * @return matrix object
- * @throws DMLRuntimeException if DMLRuntimeException occurs
- */
- public MatrixObject getSparseMatrixOutputForGPUInstruction(String varName, long nnz)
- throws DMLRuntimeException
- {
- MatrixObject mo = allocateGPUMatrixObject(varName);
- mo.getMatrixCharacteristics().setNonZeros(nnz);
- mo.getGPUObject().acquireDeviceModifySparse();
- return mo;
- }
-
- /**
* Allocates the {@link GPUObject} for a given LOPS Variable (eg. _mVar3)
* @param varName variable name
* @return matrix object
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index 8a65267..a9ca2c3 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -50,7 +50,6 @@ import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyze
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.spark.CheckpointSPInstruction;
import org.apache.sysml.runtime.instructions.spark.SPInstruction;
-import org.apache.sysml.runtime.instructions.spark.data.BlockPartitioner;
import org.apache.sysml.runtime.instructions.spark.data.BroadcastObject;
import org.apache.sysml.runtime.instructions.spark.data.LineageObject;
import org.apache.sysml.runtime.instructions.spark.data.PartitionedBlock;
@@ -610,17 +609,6 @@ public class SparkExecutionContext extends ExecutionContext
return bret;
}
- public BlockPartitioner getPartitionerForRDDVariable(String varname)
- throws DMLRuntimeException
- {
- //get input rdd and matrix characteristics
- JavaPairRDD<MatrixIndexes,MatrixBlock> in = getBinaryBlockRDDHandleForVariable(varname);
- MatrixCharacteristics mc = getMatrixCharacteristics(varname);
-
- //create tile-based matrix partitioner
- return new BlockPartitioner(mc, in.partitions().size());
- }
-
/**
* Keep the output rdd of spark rdd operations as meta data of matrix/frame
* objects in the symbol table.
[2/3] incubator-systemml git commit: [SYSTEMML-1153] Remove unused
methods from project
Posted by de...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
index 22830cf..e146821 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
@@ -57,15 +57,7 @@ public class LocalParWorker extends ParWorker implements Runnable
_stopped = false;
_max_retry = max_retry;
}
-
- /**
- * Sets the status to stopped such that execution will be aborted as soon as the
- * current task is finished.
- */
- public void setStopped() {
- _stopped = true;
- }
-
+
public void setFunctionNames(Collection<String> fnNames) {
_fnNames = fnNames;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
index 1ebaf00..dc4ca64 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
@@ -106,18 +106,6 @@ public class LocalTaskQueue<T>
_closedInput = true;
notifyAll(); //notify all waiting readers
}
-
- /**
- * Synchronized read of the current number of tasks in the queue.
- *
- * @return number of tasks in queue
- * @throws InterruptedException if InterruptedException occurs
- */
- public synchronized int size()
- throws InterruptedException
- {
- return _data.size();
- }
@Override
public synchronized String toString()
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
index 5c5c5a6..e0b30f4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
@@ -107,12 +107,6 @@ public abstract class ParWorker
return _numIters;
}
- public void resetExecutedTasks()
- {
- _numTasks = 0;
- _numIters = 0;
- }
-
protected void pinResultVariables()
{
for( String var : _resultVars )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index bc6370d..3ac5722 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -323,23 +323,6 @@ public class ProgramConverter
return tmpPB;
}
- public static ForProgramBlock createShallowCopyParForProgramBlock(ParForProgramBlock pfpb, Program prog )
- throws DMLRuntimeException
- {
- ParForProgramBlock tmpPB = new ParForProgramBlock(prog,pfpb.getIterablePredicateVars(),pfpb.getParForParams());
-
- tmpPB.setStatementBlock( pfpb.getStatementBlock() );
- tmpPB.setResultVariables( pfpb.getResultVariables() );
-
- tmpPB.setFromInstructions( pfpb.getFromInstructions() );
- tmpPB.setToInstructions( pfpb.getToInstructions() );
- tmpPB.setIncrementInstructions( pfpb.getIncrementInstructions() );
- tmpPB.setExitInstructions( pfpb.getExitInstructions() );
- tmpPB.setChildBlocks( pfpb.getChildBlocks() );
-
- return tmpPB;
- }
-
public static ParForProgramBlock createDeepCopyParForProgramBlock(ParForProgramBlock pfpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
index 0934ecf..de6a9d0 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java
@@ -87,24 +87,6 @@ public class Task implements Serializable
return _iterations.size();
}
- public void mergeTask( Task task )
- {
- //check for set iteration type
- if( _type==TaskType.RANGE )
- throw new RuntimeException("Task Merging not supported for tasks of type ITERATION_RANGE.");
-
- //check for same iteration name
- String var1 = _iterations.getFirst().getName();
- String var2 = task._iterations.getFirst().getName();
- if( !var1.equals(var2) )
- throw new RuntimeException("Task Merging not supported for tasks with different variable names");
-
- //merge tasks
- for( IntObject o : task._iterations )
- _iterations.addLast( o );
- }
-
-
@Override
public String toString()
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
index 476522a..f76d1ef 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
@@ -178,22 +178,6 @@ public abstract class CostEstimator
return val;
}
- public double computeLocalParBound(OptTree plan, OptNode n)
- {
- return Math.floor(rComputeLocalValueBound(plan.getRoot(), n, plan.getCK()));
- }
-
- public double computeLocalMemoryBound(OptTree plan, OptNode n)
- {
- return rComputeLocalValueBound(plan.getRoot(), n, plan.getCM());
- }
-
- public double getMinMemoryUsage(OptNode pn)
- {
- // TODO implement for DP enum optimizer
- throw new RuntimeException("Not implemented yet.");
- }
-
protected double getDefaultEstimate(TestMeasure measure)
{
double val = -1;
@@ -239,43 +223,4 @@ public abstract class CostEstimator
ret /= len; //weighting
return ret;
}
-
- protected double rComputeLocalValueBound( OptNode current, OptNode node, double currentVal )
- {
- if( current == node ) //found node
- return currentVal;
- else if( current.isLeaf() ) //node not here
- return -1;
- else
- {
- switch( current.getNodeType() )
- {
- case GENERIC:
- case FUNCCALL:
- case IF:
- case WHILE:
- case FOR:
- for( OptNode c : current.getChilds() )
- {
- double lval = rComputeLocalValueBound(c, node, currentVal);
- if( lval > 0 )
- return lval;
- }
- break;
- case PARFOR:
- for( OptNode c : current.getChilds() )
- {
- double lval = rComputeLocalValueBound(c, node, currentVal/current.getK());
- if( lval > 0 )
- return lval;
- }
- break;
- default:
- //do nothing
- }
- }
-
- return -1;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
index 4628c8c..1a5d8f4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
@@ -79,19 +79,6 @@ public class CostEstimatorRuntime extends CostEstimator
return getLeafNodeEstimate(measure, node);
}
- public double getEstimate( TestMeasure measure, String instName, double datasize, double sparsity, DataFormat dataformat )
- throws DMLRuntimeException
- {
- return getEstimate(measure, instName, datasize, sparsity, DEFAULT_EST_PARALLELISM, dataformat);
- }
-
- public double getEstimate( TestMeasure measure, String instName, double datasize, double sparsity, double parallelism, DataFormat dataformat )
- throws DMLRuntimeException
- {
- double dim = Math.sqrt( datasize );
- return getEstimate(measure, instName, dim, dim, dim, sparsity, parallelism, dataformat);
- }
-
public double getEstimate( TestMeasure measure, String instName, double dim1, double dim2, double dim3, double sparsity, DataFormat dataformat )
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
index a317de2..271d018 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
@@ -25,9 +25,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.Set;
-import org.apache.sysml.lops.LopProperties;
import org.apache.sysml.lops.Lop;
-
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
@@ -55,17 +53,7 @@ public class OptNode
CP,
MR,
SPARK;
-
- public LopProperties.ExecType toLopsExecType() {
- switch( this ) {
- case CP: return LopProperties.ExecType.CP;
- case MR: return LopProperties.ExecType.MR;
- case SPARK: return LopProperties.ExecType.SPARK;
- }
-
- return null;
- }
-
+
public ParForProgramBlock.PExecMode toParForExecMode() {
switch( this ) {
case CP: return ParForProgramBlock.PExecMode.LOCAL;
@@ -261,30 +249,6 @@ public class OptNode
return ret;
}
- public boolean containsNode( OptNode qn )
- {
- boolean ret = (this == qn);
- if( !ret && !isLeaf() )
- for( OptNode n : _childs ) {
- ret |= n.containsNode(qn);
- if( ret ) break; //early abort
- }
-
- return ret;
- }
-
- public boolean containsNode( NodeType type )
- {
- boolean ret = (_ntype == type);
- if( !ret && !isLeaf() )
- for( OptNode n : _childs ) {
- ret |= n.containsNode(type);
- if( ret ) break; //early abort
- }
-
- return ret;
- }
-
public boolean isLeaf()
{
return ( _childs == null || _childs.isEmpty() );
@@ -626,23 +590,4 @@ public class OptNode
return max;
}
- @SuppressWarnings("unchecked")
- public OptNode createShallowClone()
- {
- OptNode n = new OptNode(_ntype,_etype);
- n.setID(_id);
- n.setK(_k);
- if( _childs != null )
- n.setChilds( (ArrayList<OptNode>)_childs.clone() );
- if( _params != null )
- n.setParams((HashMap<ParamType,String>)_params.clone());
- return n;
- }
-
- public OptNode createDeepClone()
- {
- throw new RuntimeException("not implemented yet");
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
index dba6dd1..71d0931 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
@@ -91,9 +91,6 @@ public class OptTreeConverter
//internal state
private static OptTreePlanMappingAbstract _hlMap = null;
private static OptTreePlanMappingRuntime _rtMap = null;
- private static OptNode _tmpParent = null;
- private static OptNode _tmpChildOld = null;
- private static OptNode _tmpChildNew = null;
static
{
@@ -126,18 +123,6 @@ public class OptTreeConverter
return tree;
}
- public static OptTree createOptTree( int ck, double cm, ParForProgramBlock pfpb )
- throws DMLRuntimeException
- {
- // TODO: Passing an empty variable map here, for now. Must be reevaluated
- // whenever this function is used.
- LocalVariableMap vars = new LocalVariableMap();
- OptNode root = rCreateOptNode( pfpb, vars, true, true );
- OptTree tree = new OptTree(ck, cm, root);
-
- return tree;
- }
-
public static OptTree createAbstractOptTree( int ck, double cm, ParForStatementBlock pfsb, ParForProgramBlock pfpb, Set<String> memo, ExecutionContext ec )
throws DMLRuntimeException
{
@@ -866,106 +851,6 @@ public class OptTreeConverter
{
return _hlMap;
}
-
- public static OptTreePlanMappingRuntime getRuntimePlanMapping()
- {
- return _rtMap;
- }
-
- public static OptNode exchangeTemporary(OptNode pRoot, long hlNodeID, OptNode newRtNode)
- throws DMLRuntimeException
- {
- OptNode hlNode = _hlMap.getOptNode(hlNodeID);
- if( hlNode.getNodeType() == NodeType.PARFOR )
- {
- ParForProgramBlock pb = (ParForProgramBlock) _hlMap.getMappedProg(hlNodeID)[1];
- OptNode rtNode = _rtMap.getOptNode(pb);
-
- //copy node internals (because it might be root node)
- _tmpChildOld = rtNode.createShallowClone();
- rtNode.setExecType(newRtNode.getExecType()); //TODO extend as required
- }
- else if (hlNode.getNodeType() == NodeType.HOP)
- {
- long pid1 = _hlMap.getMappedParentID(hlNode.getID()); //pbID
- ProgramBlock pb = (ProgramBlock) _hlMap.getMappedProg(pid1)[1];
- OptNode rtNode1 = _rtMap.getOptNode(pb);
- long pid2 = _rtMap.getMappedParentID(rtNode1.getID());
- OptNode rtNode2 = _rtMap.getOptNode(pid2);
-
- _tmpParent = rtNode2;
- _tmpChildOld = rtNode1;
- _tmpChildNew = newRtNode;
- _tmpParent.exchangeChild(_tmpChildOld, _tmpChildNew);
- }
- else
- {
- throw new DMLRuntimeException("Unexpected node type for plan node exchange.");
- }
-
- return pRoot;
- }
-
- public static void revertTemporaryChange( long hlNodeID )
- throws DMLRuntimeException
- {
- OptNode node = _hlMap.getOptNode(hlNodeID);
-
- if( node.getNodeType() == NodeType.PARFOR )
- {
- ParForProgramBlock pb = (ParForProgramBlock) _hlMap.getMappedProg(hlNodeID)[1];
- OptNode rtNode = _rtMap.getOptNode(pb);
- rtNode.setExecType(_tmpChildOld.getExecType());
- }
- else if( node.getNodeType() == NodeType.HOP )
- {
- //revert change (overwrite tmp child)
- _tmpParent.exchangeChild(_tmpChildNew,_tmpChildOld);
- }
- else
- {
- throw new DMLRuntimeException("Unexpected node type for plan node exchange.");
- }
-
- //cleanup
- _tmpParent = null;
- _tmpChildOld = null;
- }
-
- public static OptNode exchangePermanently(OptNode pRoot, long hlNodeID, OptNode newRtNode)
- throws DMLRuntimeException
- {
- OptNode hlNode = _hlMap.getOptNode(hlNodeID);
- if( hlNode.getNodeType() == NodeType.PARFOR )
- {
- ParForProgramBlock pb = (ParForProgramBlock) _hlMap.getMappedProg(hlNodeID)[1];
- OptNode rtNode = _rtMap.getOptNode(pb);
-
- //copy node internals (because it might be root node)
- //(no need for update mapping)
- rtNode.setExecType(newRtNode.getExecType()); //
- }
- else if (hlNode.getNodeType() == NodeType.HOP)
- {
- long pid1 = _hlMap.getMappedParentID(hlNode.getID()); //pbID
- ProgramBlock pb = (ProgramBlock) _hlMap.getMappedProg(pid1)[1];
- OptNode rtNode1 = _rtMap.getOptNode(pb);
- long pid2 = _rtMap.getMappedParentID(rtNode1.getID());
- OptNode rtNode2 = _rtMap.getOptNode(pid2);
-
- rtNode2.exchangeChild(rtNode1, newRtNode);
-
- //finally update mapping (all internal repositories)
- newRtNode.setID(rtNode1.getID());
- _rtMap.replaceMapping(pb, newRtNode);
- }
- else
- {
- throw new DMLRuntimeException("Unexpected node type for plan node exchange.");
- }
-
- return pRoot;
- }
public static void clear()
{
@@ -973,10 +858,6 @@ public class OptTreeConverter
_hlMap.clear();
if( _rtMap != null )
_rtMap.clear();
-
- _tmpParent = null;
- _tmpChildOld = null;
- _tmpChildNew = null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
index 8efc561..0a7d9f6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
@@ -21,7 +21,6 @@ package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.HashMap;
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.sysml.runtime.controlprogram.ProgramBlock;
import org.apache.sysml.runtime.instructions.Instruction;
@@ -70,15 +69,7 @@ public class OptTreePlanMappingRuntime extends OptTreePlanMapping
{
return _id_rtprog.get( id );
}
-
- public OptNode getOptNode( Object prog )
- {
- for( Entry<Long,Object> e : _id_rtprog.entrySet() )
- if( e.getValue() == prog )
- return _id_optnode.get(e.getKey());
- return null;
- }
-
+
@Override
public void clear()
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
index 9a2d7dc..a7064da 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
@@ -20,9 +20,7 @@
package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.Map.Entry;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -33,35 +31,22 @@ import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.ipa.InterProceduralAnalysis;
+import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.hops.rewrite.HopRewriteRule;
import org.apache.sysml.hops.rewrite.ProgramRewriteStatus;
import org.apache.sysml.hops.rewrite.ProgramRewriter;
import org.apache.sysml.hops.rewrite.RewriteConstantFolding;
import org.apache.sysml.hops.rewrite.RewriteRemoveUnnecessaryBranches;
import org.apache.sysml.hops.rewrite.StatementBlockRewriteRule;
-import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.ForStatement;
-import org.apache.sysml.parser.ForStatementBlock;
-import org.apache.sysml.parser.IfStatement;
-import org.apache.sysml.parser.IfStatementBlock;
-import org.apache.sysml.parser.LanguageException;
import org.apache.sysml.parser.ParForStatementBlock;
-import org.apache.sysml.parser.StatementBlock;
-import org.apache.sysml.parser.WhileStatement;
-import org.apache.sysml.parser.WhileStatementBlock;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
-import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
-import org.apache.sysml.runtime.controlprogram.Program;
-import org.apache.sysml.runtime.controlprogram.ProgramBlock;
-import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysml.runtime.controlprogram.parfor.opt.Optimizer.CostModelType;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
@@ -102,48 +87,6 @@ public class OptimizationWrapper
.setLevel((Level) Level.DEBUG);
}
}
-
- /**
- * Called once per DML script (during program compile time)
- * in order to optimize all top-level parfor program blocks.
- *
- * NOTE: currently note used at all.
- *
- * @param prog dml program
- * @param rtprog runtime program
- * @param monitor ?
- * @throws DMLRuntimeException if DMLRuntimeException occurs
- * @throws LanguageException if LanguageException occurs
- */
- public static void optimize(DMLProgram prog, Program rtprog, boolean monitor)
- throws DMLRuntimeException, LanguageException
- {
- LOG.debug("ParFOR Opt: Running optimize all on DML program "+DMLScript.getUUID());
-
- //init internal structures
- HashMap<Long, ParForStatementBlock> sbs = new HashMap<Long, ParForStatementBlock>();
- HashMap<Long, ParForProgramBlock> pbs = new HashMap<Long, ParForProgramBlock>();
-
- //find all top-level paror pbs
- findParForProgramBlocks(prog, rtprog, sbs, pbs);
-
- // Create an empty symbol table
- ExecutionContext ec = ExecutionContextFactory.createContext();
-
- //optimize each top-level parfor pb independently
- for( Entry<Long, ParForProgramBlock> entry : pbs.entrySet() )
- {
- long key = entry.getKey();
- ParForStatementBlock sb = sbs.get(key);
- ParForProgramBlock pb = entry.getValue();
-
- //optimize (and implicit exchange)
- POptMode type = pb.getOptimizationMode(); //known to be >0
- optimize( type, sb, pb, ec, monitor );
- }
-
- LOG.debug("ParFOR Opt: Finished optimization for DML program "+DMLScript.getUUID());
- }
/**
* Called once per top-level parfor (during runtime, on parfor execute)
@@ -337,88 +280,6 @@ public class OptimizationWrapper
}
}
- private static void findParForProgramBlocks( DMLProgram prog, Program rtprog,
- HashMap<Long, ParForStatementBlock> sbs, HashMap<Long, ParForProgramBlock> pbs )
- throws LanguageException
- {
- //handle function program blocks
- HashMap<String,FunctionProgramBlock> fpbs = rtprog.getFunctionProgramBlocks();
- for( Entry<String, FunctionProgramBlock> entry : fpbs.entrySet() )
- {
- String[] keypart = entry.getKey().split( Program.KEY_DELIM );
- String namespace = keypart[0];
- String name = keypart[1];
-
- ProgramBlock pb = entry.getValue();
- StatementBlock sb = prog.getFunctionStatementBlock(namespace, name);
-
- //recursive find
- rfindParForProgramBlocks(sb, pb, sbs, pbs);
- }
-
- //handle actual program blocks
- ArrayList<ProgramBlock> tpbs = rtprog.getProgramBlocks();
- for( int i=0; i<tpbs.size(); i++ )
- {
- ProgramBlock pb = tpbs.get(i);
- StatementBlock sb = prog.getStatementBlock(i);
-
- //recursive find
- rfindParForProgramBlocks(sb, pb, sbs, pbs);
- }
- }
-
- private static void rfindParForProgramBlocks( StatementBlock sb, ProgramBlock pb,
- HashMap<Long, ParForStatementBlock> sbs, HashMap<Long, ParForProgramBlock> pbs )
- {
- if( pb instanceof ParForProgramBlock )
- {
- //put top-level parfor into map, but no recursion
- ParForProgramBlock pfpb = (ParForProgramBlock) pb;
- ParForStatementBlock pfsb = (ParForStatementBlock) sb;
-
- LOG.trace("ParFOR: found ParForProgramBlock with POptMode="+pfpb.getOptimizationMode().toString());
-
- if( pfpb.getOptimizationMode() != POptMode.NONE )
- {
- //register programblock tree for optimization
- long pfid = pfpb.getID();
- pbs.put(pfid, pfpb);
- sbs.put(pfid, pfsb);
- }
- }
- else if( pb instanceof ForProgramBlock )
- {
- //recursive find
- ArrayList<ProgramBlock> fpbs = ((ForProgramBlock) pb).getChildBlocks();
- ArrayList<StatementBlock> fsbs = ((ForStatement)((ForStatementBlock) sb).getStatement(0)).getBody();
- for( int i=0; i< fpbs.size(); i++ )
- rfindParForProgramBlocks(fsbs.get(i), fpbs.get(i), sbs, pbs);
- }
- else if( pb instanceof WhileProgramBlock )
- {
- //recursive find
- ArrayList<ProgramBlock> wpbs = ((WhileProgramBlock) pb).getChildBlocks();
- ArrayList<StatementBlock> wsbs = ((WhileStatement)((WhileStatementBlock) sb).getStatement(0)).getBody();
- for( int i=0; i< wpbs.size(); i++ )
- rfindParForProgramBlocks(wsbs.get(i), wpbs.get(i), sbs, pbs);
- }
- else if( pb instanceof IfProgramBlock )
- {
- //recursive find
- IfProgramBlock ifpb = (IfProgramBlock) pb;
- IfStatement ifs = (IfStatement) ((IfStatementBlock) sb).getStatement(0);
- ArrayList<ProgramBlock> ipbs1 = ifpb.getChildBlocksIfBody();
- ArrayList<ProgramBlock> ipbs2 = ifpb.getChildBlocksElseBody();
- ArrayList<StatementBlock> isbs1 = ifs.getIfBody();
- ArrayList<StatementBlock> isbs2 = ifs.getElseBody();
- for( int i=0; i< ipbs1.size(); i++ )
- rfindParForProgramBlocks(isbs1.get(i), ipbs1.get(i), sbs, pbs);
- for( int i=0; i< ipbs2.size(); i++ )
- rfindParForProgramBlocks(isbs2.get(i), ipbs2.get(i), sbs, pbs);
- }
- }
-
private static Optimizer createOptimizer( POptMode otype )
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
index fc71971..fdeffb8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
@@ -19,20 +19,13 @@
package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import java.util.Collection;
-import java.util.LinkedList;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-import org.apache.sysml.hops.Hop;
import org.apache.sysml.parser.ParForStatementBlock;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.POptMode;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
/**
@@ -102,58 +95,5 @@ public abstract class Optimizer
{
return _numEvaluatedPlans;
}
-
-
-
-
-
- ///////
- //methods for common basic primitives
-
- /**
- * Enum node plans (only for current opt node)
- *
- * @param n internal representation of a plan alternative for program blocks and instructions
- * @param lck ?
- * @return collection of optimization nodes
- */
- protected Collection<OptNode> enumPlans( OptNode n, double lck )
- {
- Collection<OptNode> plans = enumerateExecTypes( n );
-
- //TODO additional enumerations / potential rewrites go here
-
- return plans;
- }
-
- private Collection<OptNode> enumerateExecTypes( OptNode n )
- {
- Collection<OptNode> dTypes = new LinkedList<OptNode>();
- boolean genAlternatives = false;
-
- //determine if alternatives should be generated
- if( n.isLeaf() ) //hop
- {
- Hop hop = OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
- if( hop.allowsAllExecTypes() )
- genAlternatives = true;
- }
- else if( n.getNodeType()==NodeType.PARFOR ) //parfor pb
- {
- genAlternatives = true;
- }
- //generate alternatives
- if( genAlternatives )
- {
- OptNode c1 = n.createShallowClone();
- OptNode c2 = n.createShallowClone();
- c1.setExecType(ExecType.CP);
- c2.setExecType(ExecType.MR);
- dTypes.add( c1 );
- dTypes.add( c2 );
- }
-
- return dTypes;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 357aff8..532c41c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -535,12 +535,6 @@ public class OptimizerRuleBased extends Optimizer
return mem;
}
- protected static LopProperties.ExecType getRIXExecType( MatrixObject mo, PDataPartitionFormat dpf )
- throws DMLRuntimeException
- {
- return getRIXExecType(mo, dpf, false);
- }
-
protected static LopProperties.ExecType getRIXExecType( MatrixObject mo, PDataPartitionFormat dpf, boolean withSparsity )
throws DMLRuntimeException
{
@@ -579,37 +573,6 @@ public class OptimizerRuleBased extends Optimizer
return LopProperties.ExecType.CP_FILE;
}
- public static PDataPartitionFormat decideBlockWisePartitioning( MatrixObject mo, PDataPartitionFormat dpf )
- throws DMLRuntimeException
- {
- long rlen = mo.getNumRows();
- long clen = mo.getNumColumns();
- long brlen = mo.getNumRowsPerBlock();
- long bclen = mo.getNumColumnsPerBlock();
- long k = InfrastructureAnalyzer.getRemoteParallelMapTasks();
-
- PDataPartitionFormat ret = dpf;
- if( getRIXExecType(mo, dpf)==LopProperties.ExecType.CP )
- if( ret == PDataPartitionFormat.ROW_WISE )
- {
- if( rlen/brlen > 4*k && //note: average sparsity, read must deal with it
- getRIXExecType(mo, PDataPartitionFormat.ROW_BLOCK_WISE, false)==LopProperties.ExecType.CP )
- {
- ret = PDataPartitionFormat.ROW_BLOCK_WISE;
- }
- }
- else if( ret == PDataPartitionFormat.COLUMN_WISE )
- {
- if( clen/bclen > 4*k && //note: average sparsity, read must deal with it
- getRIXExecType(mo, PDataPartitionFormat.COLUMN_BLOCK_WISE, false)==LopProperties.ExecType.CP )
- {
- ret = PDataPartitionFormat.COLUMN_BLOCK_WISE;
- }
- }
-
- return ret;
- }
-
public static boolean allowsBinaryCellPartitions( MatrixObject mo, PDataPartitionFormat dpf )
throws DMLRuntimeException
{
@@ -3550,12 +3513,7 @@ public class OptimizerRuleBased extends Optimizer
{
this.hopCandidate = hopCandidate;
}
-
- ProgramBlock getProgramBlock()
- {
- return pb;
- }
-
+
int getLocation()
{
return this.iLocation;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
index 5d43941..9635707 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
@@ -30,9 +30,9 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.StringTokenizer;
-import java.util.Map.Entry;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLOutputFactory;
@@ -43,10 +43,6 @@ import javax.xml.stream.XMLStreamWriter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-import au.com.bytecode.opencsv.CSVReader;
-import au.com.bytecode.opencsv.CSVWriter;
-
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
@@ -54,10 +50,10 @@ import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DataIdentifier;
-import org.apache.sysml.parser.ExternalFunctionStatement;
-import org.apache.sysml.parser.ParseException;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
@@ -83,9 +79,11 @@ import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.runtime.util.LocalFileUtils;
import org.apache.sysml.runtime.util.MapReduceTool;
+import au.com.bytecode.opencsv.CSVReader;
+import au.com.bytecode.opencsv.CSVWriter;
+
/**
* DML Instructions Performance Test Tool:
*
@@ -289,12 +287,6 @@ public class PerfTestTool
return tmp;
}
- public CostFunction getInvariantCostFunction( TestMeasure measure, TestVariable[] variable, DataFormat dataformat )
- {
- //TODO: implement for additional rewrites
- throw new RuntimeException("Not implemented yet.");
- }
-
@SuppressWarnings("all")
public static boolean runTest()
{
@@ -920,44 +912,6 @@ public class PerfTestTool
return data;
}
- public static MatrixObject generateInputDataset(String fname, double datasize, double sparsity, DataFormat df)
- throws IOException, CacheException
- {
- int dim = (int)Math.sqrt( datasize );
-
- //create random test data
- double[][] d = generateTestMatrix(dim, dim, 1, 100, sparsity, 7);
-
- //create matrix block
- MatrixBlock mb = null;
- switch( df )
- {
- case DENSE:
- mb = new MatrixBlock(dim,dim,false);
- break;
- case SPARSE:
- mb = new MatrixBlock(dim,dim,true, (int)(sparsity*dim*dim));
- break;
- }
-
- //insert data
- for(int i=0; i < dim; i++)
- for(int j=0; j < dim; j++)
- if( d[i][j]!=0 )
- mb.setValue(i, j, d[i][j]);
-
- MapReduceTool.deleteFileIfExistOnHDFS(fname);
-
- MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
- MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
- mo.acquireModify(mb);
- mo.release();
- mo.exportData(); //write to HDFS
-
- return mo;
- }
-
public static MatrixObject generateInputDataset(String fname, double dim1, double dim2, double sparsity, DataFormat df)
throws IOException, CacheException
{
@@ -999,30 +953,6 @@ public class PerfTestTool
return mo;
}
- public static MatrixObject generateEmptyResult(String fname, double datasize, DataFormat df )
- throws IOException, CacheException
- {
- int dim = (int)Math.sqrt( datasize );
-
- /*
- MatrixBlock mb = null;
- switch( df )
- {
- case DENSE:
- mb = new MatrixBlock(dim,dim,false);
- break;
- case SPARSE:
- mb = new MatrixBlock(dim,dim,true);
- break;
- }*/
-
- MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
- MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
-
- return mo;
- }
-
public static MatrixObject generateEmptyResult(String fname, double dim1, double dim2, DataFormat df )
throws IOException, CacheException
{
@@ -1080,19 +1010,6 @@ public class PerfTestTool
return matrix;
}
- public static void externalReadProfile( String fname )
- throws DMLRuntimeException, XMLStreamException, IOException
- {
- //validate external name (security issue)
- if( !LocalFileUtils.validateExternalFilename(fname, false) )
- throw new DMLRuntimeException("Invalid (non-trustworthy) external profile filename.");
-
- //register internals and read external profile
- registerTestConfigurations();
- registerInstructions();
- readProfile( fname );
- }
-
@SuppressWarnings("all")
private static HashMap<Integer,Long> writeResults( String dirname )
throws IOException, DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
index ef2d880..c21d0f8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
@@ -25,15 +25,14 @@ import java.util.ArrayList;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.VisitStatus;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.IndexingOp;
import org.apache.sysml.hops.OptimizerUtils;
-import org.apache.sysml.hops.Hop.VisitStatus;
import org.apache.sysml.hops.recompile.Recompiler;
-import org.apache.sysml.lops.LopProperties;
import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties;
import org.apache.sysml.lops.LopsException;
-import org.apache.sysml.lops.compile.Dag;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.ForStatement;
@@ -47,13 +46,10 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
-import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.ProgramBlock;
import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
import org.apache.sysml.runtime.instructions.Instruction;
import org.apache.sysml.runtime.instructions.cp.ArithmeticBinaryCPInstruction;
import org.apache.sysml.runtime.instructions.cp.Data;
@@ -486,157 +482,4 @@ public class ProgramRecompiler
return tmp;
}
-
-
-
-
- /////////////////////////////////
- // experimental functionality
- //////////
-
- protected static void recompilePartialPlan( OptNode n )
- throws DMLRuntimeException
- {
- //NOTE: need to recompile complete programblock because (1) many to many relationships
- //between hops and instructions and (2) due to changed internal variable names
-
- try
- {
- //get parent program and statement block
- OptTreePlanMappingAbstract map = OptTreeConverter.getAbstractPlanMapping();
- long pid = map.getMappedParentID(n.getID());
- Object[] o = map.getMappedProg(pid);
- StatementBlock sbOld = (StatementBlock) o[0];
- ProgramBlock pbOld = (ProgramBlock) o[1];
-
- //get changed node and set type appropriately
- Hop hop = (Hop) map.getMappedHop(n.getID());
- hop.setForcedExecType(n.getExecType().toLopsExecType());
- hop.setLops(null); //to enable fresh construction
-
- //get all hops of statement and construct new instructions
- Dag<Lop> dag = new Dag<Lop>();
- for( Hop hops : sbOld.get_hops() )
- {
- hops.resetVisitStatus();
- Recompiler.rClearLops(hops);
- Lop lops = hops.constructLops();
- lops.addToDag(dag);
- }
-
- //construct new instructions
- ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getDMLConfig());
-
-
- //exchange instructions
- pbOld.getInstructions().clear();
- pbOld.getInstructions().addAll(newInst);
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
- }
-
-
- /**
- * NOTE: need to recompile complete programblock because (1) many to many relationships
- * between hops and instructions and (2) due to changed internal variable names
- *
- * @param n internal representation of a plan alternative for program blocks and instructions
- * @return program block
- * @throws DMLRuntimeException if DMLRuntimeException occurs
- */
- protected static ProgramBlock recompile( OptNode n )
- throws DMLRuntimeException
- {
- ProgramBlock pbNew = null;
-
- try
- {
- if( n.getNodeType() == NodeType.HOP )
- {
- //get parent program and statement block
- OptTreePlanMappingAbstract map = OptTreeConverter.getAbstractPlanMapping();
- long pid = map.getMappedParentID(n.getID());
- Object[] o = map.getMappedProg(pid);
- StatementBlock sbOld = (StatementBlock) o[0];
- ProgramBlock pbOld = (ProgramBlock) o[1];
- LopProperties.ExecType oldtype = null;
-
- //get changed node and set type appropriately
- Hop hop = (Hop) map.getMappedHop(n.getID());
- hop.setForcedExecType(n.getExecType().toLopsExecType());
- hop.setLops(null); //to enable fresh construction
-
- //get all hops of statement and construct new lops
- Dag<Lop> dag = new Dag<Lop>();
- for( Hop hops : sbOld.get_hops() )
- {
- hops.resetVisitStatus();
- Recompiler.rClearLops(hops);
- Lop lops = hops.constructLops();
- lops.addToDag(dag);
- }
-
- //construct new instructions
- ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getDMLConfig());
-
- //exchange instructions
- pbNew = new ProgramBlock(pbOld.getProgram());
- pbNew.setInstructions(newInst);
-
- //reset type global repository
- hop.setForcedExecType(oldtype);
-
- }
- else if( n.getNodeType() == NodeType.PARFOR )
- {
- //no recompilation required
- OptTreePlanMappingAbstract map = OptTreeConverter.getAbstractPlanMapping();
- ParForProgramBlock pb = (ParForProgramBlock)map.getMappedProg(n.getID())[1];
- pbNew = ProgramConverter.createShallowCopyParForProgramBlock(pb, pb.getProgram());
- ((ParForProgramBlock)pbNew).setExecMode(n.getExecType().toParForExecMode());
- }
- else
- {
- throw new DMLRuntimeException("Unexpected node type.");
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(ex);
- }
-
- return pbNew;
- }
-
- protected static void exchangeProgram(long hlNodeID, ProgramBlock pbNew)
- throws DMLRuntimeException
- {
- OptTreePlanMappingAbstract map = OptTreeConverter.getAbstractPlanMapping();
- OptNode node = map.getOptNode(hlNodeID);
-
- if( node.getNodeType() == NodeType.HOP )
- {
- long pid = map.getMappedParentID(hlNodeID);
- Object[] o = map.getMappedProg(pid);
- ProgramBlock pbOld = (ProgramBlock) o[1];
-
- //exchange instructions (save version)
- pbOld.getInstructions().clear();
- pbOld.getInstructions().addAll( pbNew.getInstructions() );
- }
- else if( node.getNodeType() == NodeType.PARFOR )
- {
- ParForProgramBlock pbOld = (ParForProgramBlock) map.getMappedProg(node.getID())[1];
- pbOld.setExecMode(((ParForProgramBlock)pbNew).getExecMode());
- //TODO extend as required
- }
- else
- {
- throw new DMLRuntimeException("Unexpected node type: "+node.getNodeType());
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
index d5f8c7d..7cb01a2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java
@@ -135,20 +135,6 @@ public class InfrastructureAnalyzer
_remoteParReduce = preduce;
}
- /**
- * Gets the totals number of available map and reduce slots.
- *
- * @return number of available remote parallel task slots
- */
- public static int getRemoteParallelTasks()
- {
- if( _remoteParMap == -1 )
- analyzeHadoopCluster();
-
- return _remoteParMap + _remoteParReduce;
- }
-
-
///////
//methods for obtaining memory properties
@@ -202,20 +188,7 @@ public class InfrastructureAnalyzer
{
_remoteJVMMaxMemReduce = remoteMem;
}
-
- /**
- * Gets the maximum memory requirement [in bytes] of a given hadoop job.
- *
- * @param job job configuration
- * @return remote max memory of hadoop job
- */
- public static long getRemoteMaxMemory( JobConf job )
- {
- return (1024*1024) * Math.max(
- job.getMemoryForMapTask(),
- job.getMemoryForReduceTask() );
- }
-
+
/**
* Gets the maximum sort buffer memory requirement [in bytes] of a hadoop task.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
index 2547709..a558e16 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/StatisticMonitor.java
@@ -118,17 +118,7 @@ public class StatisticMonitor
stats.addLast(s);
}
-
- /**
- * Cleans up the whole repository by discarding all collected information.
- */
- public static void cleanUp()
- {
- _mapPwPf.clear();
- _pfstats.clear();
- _pwstats.clear();
- }
-
+
/**
* Globally disables statistic monitor for the currently activ JVM.
*/
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
index f494b88..e70ae9c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/util/IDHandler.java
@@ -33,31 +33,6 @@ import java.net.InetAddress;
*/
public class IDHandler
{
-
- public static long extractUncheckedLongID( String taskID )
- {
- //in: e.g., task_local_0002_m_000009 or jobID + ...
- //out: e.g., 2000009
-
- //generic parsing for flexible taskID formats
- char[] c = taskID.toCharArray(); //all chars
- long value = 1; //1 catch leading zeros as well
- for( int i=0; i<c.length; i++ )
- {
- if( c[i] >= 48 && c[i]<=57 ) //'0'-'9'
- {
- long newVal = (c[i]-48);
-
- if( (Long.MAX_VALUE-value*10) < newVal )
- throw new RuntimeException("WARNING: extractLongID will produced numeric overflow "+value);
-
- value = value*10 + newVal;
- }
- }
-
- return value;
- }
-
public static int extractIntID( String taskID )
{
int maxlen = (int)(Math.log10(Integer.MAX_VALUE));
@@ -66,13 +41,6 @@ public class IDHandler
}
- public static long extractLongID( String taskID )
- {
- int maxlen = (int)(Math.log10(Long.MAX_VALUE));
- long longVal = extractID( taskID, maxlen );
- return longVal;
- }
-
public static long concatIntIDsToLong( int part1, int part2 )
{
//big-endian version (in java uses only big endian)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
index e98b46e..dad8c2d 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/Builtin.java
@@ -269,49 +269,7 @@ public class Builtin extends ValueFunction
// cloning is not supported for singleton classes
throw new CloneNotSupportedException();
}
-
- public boolean checkArity(int _arity) throws DMLRuntimeException {
- switch (bFunc) {
- case ABS:
- case SIN:
- case COS:
- case TAN:
- case ASIN:
- case ACOS:
- case ATAN:
- case SIGN:
- case SQRT:
- case EXP:
- case PLOGP:
- case NROW:
- case NCOL:
- case LENGTH:
- case ROUND:
- case PRINT:
- case MAXINDEX:
- case MININDEX:
- case STOP:
- case CEIL:
- case FLOOR:
- case CUMSUM:
- case INVERSE:
- case SPROP:
- case SIGMOID:
- case SELP:
- return (_arity == 1);
-
- case LOG:
- case LOG_NZ:
- return (_arity == 1 || _arity == 2);
-
- case MAX:
- case MIN:
- return (_arity == 2);
- default:
- throw new DMLRuntimeException("checkNumberOfOperands(): Unknown opcode: " + bFunc);
- }
- }
-
+
public double execute (double in)
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java b/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
index dc15a4d..afd5483 100644
--- a/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
+++ b/src/main/java/org/apache/sysml/runtime/functionobjects/OffsetColumnIndex.java
@@ -43,12 +43,7 @@ public class OffsetColumnIndex extends IndexFunction
// singleObj = new OffsetColumnIndex(offset);
//return singleObj;
}
-
- public void setOutputSize(int rows, int columns){
- numRowsInOutput = rows;
- numColumnsInOutput = columns;
- }
-
+
public void setOffset(int offset){
this.offset = offset;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
index a18d49b..6ef39f1 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
@@ -212,13 +212,7 @@ public class InstructionUtils
Builtin.BuiltinCode bfc = Builtin.String2BuiltinCode.get(opcode);
return (bfc != null);
}
-
- public static boolean isOperand(String str) {
- //note: split required for empty tokens
- String[] parts = str.split(Instruction.DATATYPE_PREFIX);
- return (parts.length > 1);
- }
-
+
/**
* Evaluates if at least one instruction of the given instruction set
* used the distributed cache; this call can also be used for individual
@@ -427,22 +421,6 @@ public class InstructionUtils
return agg;
}
- public static AggregateUnaryOperator parseCumulativeAggregateUnaryOperator(UnaryOperator uop)
- {
- Builtin f = (Builtin)uop.fn;
-
- if( f.getBuiltinCode()==BuiltinCode.CUMSUM )
- return parseCumulativeAggregateUnaryOperator("ucumack+") ;
- else if( f.getBuiltinCode()==BuiltinCode.CUMPROD )
- return parseCumulativeAggregateUnaryOperator("ucumac*") ;
- else if( f.getBuiltinCode()==BuiltinCode.CUMMIN )
- return parseCumulativeAggregateUnaryOperator("ucumacmin") ;
- else if( f.getBuiltinCode()==BuiltinCode.CUMMAX )
- return parseCumulativeAggregateUnaryOperator("ucumacmax" ) ;
-
- throw new RuntimeException("Unsupported cumulative aggregate unary operator: "+f.getBuiltinCode());
- }
-
public static AggregateUnaryOperator parseBasicCumulativeAggregateUnaryOperator(UnaryOperator uop)
{
Builtin f = (Builtin)uop.fn;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
index 5b3c7e9..780325b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
@@ -389,136 +389,6 @@ public class MRJobInstruction extends Instruction
setReplication(replication);
}
- public void setRandInstructions(long [] numRows, String[] inLabels,
- String randInstructions, String mapperInstructions,
- String aggInstructions, String otherInstructions, String [] outLabels, byte [] resultIndex,
- int numReducers, int replication)
- {
- setOutputs(resultIndex);
-
- setRecordReaderInstructions("");
- setRandInstructions(randInstructions);
- setMapperInstructions(mapperInstructions);
- setShuffleInstructions("");
- setAggregateInstructionsInReducer(aggInstructions);
- setOtherInstructionsInReducer(otherInstructions);
-
- setInputOutputLabels(inLabels, outLabels);
-
- setNumberOfReducers(numReducers);
- setReplication(replication);
- }
-
-
- public void setMMCJInstructions(String[] inLabels,
- String mapperInstructions, String shuffleInstructions,
- String [] outLabels, byte [] resultIndex,
- int numReducers, int replication)
- {
- setOutputs(resultIndex);
-
- setMapperInstructions(mapperInstructions);
- setShuffleInstructions(shuffleInstructions);
- setAggregateInstructionsInReducer("");
- setOtherInstructionsInReducer("");
-
- setInputOutputLabels(inLabels, outLabels);
-
- setNumberOfReducers(numReducers);
- setReplication(replication);
- }
-
- public void setMMRJInstructions(String[] inLabels,
- String mapperInstructions, String shuffleInstructions, String aggInstructions, String otherInstructions,
- String [] outLabels, byte [] resultIndex,
- int numReducers, int replication)
- {
- setOutputs(resultIndex);
-
- setMapperInstructions(mapperInstructions);
- setShuffleInstructions(shuffleInstructions);
- setAggregateInstructionsInReducer(aggInstructions);
- setOtherInstructionsInReducer(otherInstructions);
-
- setInputOutputLabels(inLabels, outLabels);
-
- setNumberOfReducers(numReducers);
- setReplication(replication);
- }
-
- // SortKeys Job does not have any instructions either in mapper or in reducer.
- // It just has two inputs
- public void setSORTKEYSInstructions(String [] inLabels,
- String mapperInstructions, String shuffleInstructions,
- String[] outLabels, byte [] resultIndex,
- int numReducers, int replication)
- {
- setOutputs(resultIndex);
-
- setMapperInstructions(mapperInstructions);
- setShuffleInstructions(shuffleInstructions);
- setAggregateInstructionsInReducer("");
- setOtherInstructionsInReducer("");
-
- setInputOutputLabels(inLabels, outLabels);
-
- setNumberOfReducers(numReducers);
- setReplication(replication);
- }
-
- public void setCombineInstructions(String[] inLabels,
- String shuffleInstructions, String[] outLabels, byte[] resultIndex,
- int numReducers, int replication)
- {
- setOutputs(resultIndex);
-
- setMapperInstructions("");
- setShuffleInstructions(shuffleInstructions);
- setAggregateInstructionsInReducer("");
- setOtherInstructionsInReducer("");
-
- setInputOutputLabels(inLabels, outLabels);
-
- setNumberOfReducers(numReducers);
- setReplication(replication);
- }
-
- public void setCentralMomentInstructions(String[] inLabels,
- String mapperInstructions, String shuffleInstructions,
- String[] outLabels, byte [] resultIndex,
- int numReducers, int replication)
- {
- setOutputs(resultIndex);
-
- setMapperInstructions(mapperInstructions);
- setShuffleInstructions(shuffleInstructions);
- setAggregateInstructionsInReducer("");
- setOtherInstructionsInReducer("");
-
- setInputOutputLabels(inLabels, outLabels);
-
- setNumberOfReducers(numReducers);
- setReplication(replication);
- }
-
- public void setGroupedAggInstructions(String[] inLabels,
- String shuffleInstructions, String otherInstructions,
- String[] outLabels, byte [] resultIndex,
- int numReducers, int replication)
- {
- setOutputs(resultIndex);
-
- setMapperInstructions("");
- setShuffleInstructions(shuffleInstructions);
- setAggregateInstructionsInReducer("");
- setOtherInstructionsInReducer(otherInstructions);
-
- setInputOutputLabels(inLabels, outLabels);
-
- setNumberOfReducers(numReducers);
- setReplication(replication);
- }
-
public void setReBlockInstructions(String[] inLabels,
String mapperInstructions, String reblockInstructions, String otherInstructions,
String[] outLabels, byte [] resultIndex,
@@ -567,89 +437,6 @@ public class MRJobInstruction extends Instruction
return sb.toString();
}
- public String getString(long [] arr)
- {
- StringBuilder sb = new StringBuilder();
- for(int i = 0; i < arr.length; i++) {
- sb.append(",");
- sb.append(Long.toString(arr[i]));
- }
-
- return sb.toString();
- }
-
- public String getString(int [] arr)
- {
- StringBuilder sb = new StringBuilder();
- for(int i = 0; i < arr.length; i++) {
- sb.append(",");
- sb.append(Integer.toString(arr[i]));
- }
-
- return sb.toString();
- }
-
- public String getString(OutputInfo[] iv_outputs)
- {
- StringBuilder sb = new StringBuilder();
- for(int i = 0 ; i < iv_outputs.length; i++) {
- if(iv_outputs[i] == OutputInfo.BinaryBlockOutputInfo){
- sb.append(", ");
- sb.append("BinaryBlockOutputInfo");
- }
- else if(iv_outputs[i] == OutputInfo.BinaryCellOutputInfo){
- sb.append(", ");
- sb.append("BinaryCellOutputInfo");
- }
- else if(iv_outputs[i] == OutputInfo.TextCellOutputInfo){
- sb.append(", ");
- sb.append("TextCellOutputInfo");
- }
- else {
- sb.append(", (");
- sb.append(iv_outputs[i].outputFormatClass);
- sb.append(",");
- sb.append(iv_outputs[i].outputKeyClass);
- sb.append(",");
- sb.append(iv_outputs[i].outputValueClass);
- sb.append(")");
- }
- }
-
- return sb.toString();
- }
-
- public String getString(InputInfo[] iv_inputs)
- {
- StringBuilder sb = new StringBuilder();
- for(int i = 0 ; i < iv_inputs.length; i++) {
- if(iv_inputs[i] == InputInfo.BinaryBlockInputInfo){
- sb.append(", ");
- sb.append("BinaryBlockInputInfo");
- }
- else if(iv_inputs[i] == InputInfo.BinaryCellInputInfo){
- sb.append(", ");
- sb.append("BinaryCellInputInfo");
- }
- else if(iv_inputs[i] == InputInfo.TextCellInputInfo) {
- sb.append(", ");
- sb.append("TextCellInputInfo");
- }
- else {
- sb.append(", (");
- sb.append(iv_inputs[i].inputFormatClass);
- sb.append(",");
- sb.append(iv_inputs[i].inputKeyClass);
- sb.append(",");
- sb.append(iv_inputs[i].inputValueClass);
- sb.append(")");
- }
- }
-
- return sb.toString();
- }
-
-
public String toString()
{
String instruction = "";
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
index 2f7fdb6..345aa7b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java
@@ -19,8 +19,8 @@
package org.apache.sysml.runtime.instructions.cp;
-import org.apache.sysml.lops.Lop;
-import org.apache.sysml.parser.Expression.*;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.instructions.Instruction;
@@ -83,12 +83,6 @@ public class CPOperand
public void setLiteral(boolean literal) {
_isLiteral = literal;
}
-
- public void split_by_value_type_prefix ( String str ) {
- String[] opr = str.split(Lop.VALUETYPE_PREFIX);
- _name = opr[0];
- _valueType = ValueType.valueOf(opr[1]);
- }
public void split(String str){
String[] opr = str.split(Instruction.VALUETYPE_PREFIX);
@@ -109,10 +103,5 @@ public class CPOperand
_valueType = ValueType.valueOf(opr[1]);
}
}
-
- public void copy(CPOperand o){
- _name = o.getName();
- _valueType = o.getValueType();
- _dataType = o.getDataType();
- }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index e7dd21a..d5ce3f7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -981,27 +981,7 @@ public class VariableCPInstruction extends CPInstruction
public static Instruction prepareCreateMatrixVariableInstruction(String varName, String fileName, boolean fNameOverride, String format) throws DMLRuntimeException {
return parseInstruction(getBasicCreateVarString(varName, fileName, fNameOverride, DataType.MATRIX, format));
}
-
- public static Instruction prepareCreateVariableInstruction(String varName, String fileName, boolean fNameOverride, DataType dt, String format, MatrixCharacteristics mc) throws DMLRuntimeException {
- StringBuilder sb = new StringBuilder();
- sb.append(getBasicCreateVarString(varName, fileName, fNameOverride, dt, format));
-
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(mc.getRows());
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(mc.getCols());
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(mc.getRowsPerBlock());
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(mc.getColsPerBlock());
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(mc.getNonZeros());
-
- String str = sb.toString();
- return parseInstruction(str);
- }
-
public static Instruction prepareCreateVariableInstruction(String varName, String fileName, boolean fNameOverride, DataType dt, String format, MatrixCharacteristics mc, UpdateType update) throws DMLRuntimeException {
StringBuilder sb = new StringBuilder();
sb.append(getBasicCreateVarString(varName, fileName, fNameOverride, dt, format));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
index f039a53..319d833 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
@@ -78,10 +78,6 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI
_sptype = SPINSTRUCTION_TYPE.MultiReturnBuiltin;
_outputs = outputs;
}
-
- public CPOperand getOutput(int i) {
- return _outputs.get(i);
- }
public static MultiReturnParameterizedBuiltinSPInstruction parseInstruction( String str )
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
index a8cda44..4d2bbd2 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/UnarySPInstruction.java
@@ -20,11 +20,9 @@
package org.apache.sysml.runtime.instructions.spark;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.functionobjects.Not;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.matrix.operators.Operator;
-import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
public abstract class UnarySPInstruction extends ComputationSPInstruction
{
@@ -90,12 +88,4 @@ public abstract class UnarySPInstruction extends ComputationSPInstruction
}
return opcode;
}
-
- static SimpleOperator getSimpleUnaryOperator(String opcode)
- throws DMLRuntimeException {
- if (opcode.equalsIgnoreCase("!"))
- return new SimpleOperator(Not.getNotFnObject());
-
- throw new DMLRuntimeException("Unknown unary operator " + opcode);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
index 63fdb82..8038157 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
@@ -69,20 +69,6 @@ public class RDDAggregateUtils
}
}
- public static JavaPairRDD<MatrixIndexes, MatrixBlock> sumByKey( JavaPairRDD<MatrixIndexes, MatrixBlock> in )
- {
- //sum of blocks per key, w/o exploitation of correction blocks
- return in.reduceByKey(
- new SumMultiBlockFunction());
- }
-
- public static JavaPairRDD<MatrixIndexes, Double> sumCellsByKey( JavaPairRDD<MatrixIndexes, Double> in )
- {
- //sum of blocks per key, w/o exploitation of corrections
- return in.reduceByKey(
- new SumDoubleCellsFunction());
- }
-
public static JavaPairRDD<MatrixIndexes, MatrixBlock> sumByKeyStable( JavaPairRDD<MatrixIndexes, MatrixBlock> in )
{
//stable sum of blocks per key, by passing correction blocks along with aggregates
@@ -144,13 +130,6 @@ public class RDDAggregateUtils
new AggregateSingleBlockFunction(aop) );
}
- public static JavaPairRDD<MatrixIndexes, MatrixBlock> aggByKey( JavaPairRDD<MatrixIndexes, MatrixBlock> in, AggregateOperator aop )
- {
- //aggregate of blocks per key, w/o exploitation of correction blocks
- return in.reduceByKey(
- new AggregateMultiBlockFunction(aop));
- }
-
public static JavaPairRDD<MatrixIndexes, MatrixBlock> aggByKeyStable( JavaPairRDD<MatrixIndexes, MatrixBlock> in, AggregateOperator aop )
{
//stable sum of blocks per key, by passing correction blocks along with aggregates
@@ -507,41 +486,6 @@ public class RDDAggregateUtils
return out;
}
}
-
- /**
- * This aggregate function uses kahan+ with corrections to aggregate input blocks; it is meant for
- * reducebykey operations where we CANNOT reuse the same correction block independent of the input
- * block indexes. Note that this aggregation function does not apply to embedded corrections.
- *
- */
- private static class SumMultiBlockFunction implements Function2<MatrixBlock, MatrixBlock, MatrixBlock>
- {
- private static final long serialVersionUID = -4015979658416853324L;
-
- private AggregateOperator _op = null;
- private MatrixBlock _corr = null;
-
- public SumMultiBlockFunction()
- {
- _op = new AggregateOperator(0, KahanPlus.getKahanPlusFnObject(), true, CorrectionLocationType.NONE);
- _corr = new MatrixBlock();
- }
-
- @Override
- public MatrixBlock call(MatrixBlock arg0, MatrixBlock arg1)
- throws Exception
- {
- //copy one input to output
- MatrixBlock out = new MatrixBlock(arg0);
-
- //aggregate other input
- _corr.reset(out.getNumRows(), out.getNumColumns());
- OperationsOnMatrixValues.incrementalAggregation(out, _corr, arg1, _op, false);
-
- return out;
- }
- }
-
/**
* Note: currently we always include the correction and use a subsequent maptopair to
@@ -584,46 +528,6 @@ public class RDDAggregateUtils
return arg0;
}
}
-
- /**
- * Note: currently we always include the correction and use a subsequent maptopair to
- * drop them at the end because during aggregation we dont know if we produce an
- * intermediate or the final aggregate.
- */
- private static class AggregateMultiBlockFunction implements Function2<MatrixBlock, MatrixBlock, MatrixBlock>
- {
- private static final long serialVersionUID = -3672377410407066396L;
-
- private AggregateOperator _op = null;
- private MatrixBlock _corr = null;
-
- public AggregateMultiBlockFunction( AggregateOperator op )
- {
- _op = op;
- _corr = new MatrixBlock();
- }
-
- @Override
- public MatrixBlock call(MatrixBlock arg0, MatrixBlock arg1)
- throws Exception
- {
- //copy one first input
- MatrixBlock out = new MatrixBlock(arg0);
-
- //aggregate second input
- _corr.reset(out.getNumRows(), out.getNumColumns());
- if(_op.correctionExists) {
- OperationsOnMatrixValues.incrementalAggregation(
- out, _corr, arg1, _op, true);
- }
- else {
- OperationsOnMatrixValues.incrementalAggregation(
- out, null, arg1, _op, true);
- }
-
- return out;
- }
- }
private static class MergeBlocksFunction implements Function2<MatrixBlock, MatrixBlock, MatrixBlock>
{
@@ -669,14 +573,4 @@ public class RDDAggregateUtils
}
}
-
- private static class SumDoubleCellsFunction implements Function2<Double, Double, Double>
- {
- private static final long serialVersionUID = -8167625566734873796L;
-
- @Override
- public Double call(Double v1, Double v2) throws Exception {
- return v1 + v2;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
index 6872b57..d53f3cf 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
@@ -21,7 +21,6 @@
package org.apache.sysml.runtime.instructions.spark.utils;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import org.apache.spark.HashPartitioner;
@@ -30,9 +29,6 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.storage.StorageLevel;
-
-import scala.Tuple2;
-
import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.spark.functions.CopyBinaryCellFunction;
@@ -47,6 +43,8 @@ import org.apache.sysml.runtime.matrix.data.Pair;
import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
import org.apache.sysml.runtime.util.UtilFunctions;
+import scala.Tuple2;
+
public class SparkUtils
{
//internal configuration
@@ -110,10 +108,6 @@ public class SparkUtils
return new Pair<Long,FrameBlock>(in._1(), in._2());
}
- public static Pair<Long,FrameBlock> toIndexedFrameBlock( Long ix, FrameBlock fb ) {
- return new Pair<Long,FrameBlock>(ix, fb);
- }
-
/**
* Indicates if the input RDD is hash partitioned, i.e., it has a partitioner
* of type {@code org.apache.spark.HashPartitioner}.
@@ -155,42 +149,6 @@ public class SparkUtils
return in.mapPartitionsToPair(new CopyBlockPairFunction(deep), true);
}
- public static MatrixBlock[] partitionIntoRowBlocks( MatrixBlock mb, int blen )
- throws DMLRuntimeException
- {
- //in-memory rowblock partitioning (according to bclen of rdd)
- int lrlen = mb.getNumRows();
- int numBlocks = (int)Math.ceil((double)lrlen/blen);
- MatrixBlock[] partBlocks = new MatrixBlock[numBlocks];
- for( int i=0; i<numBlocks; i++ )
- {
- MatrixBlock tmp = new MatrixBlock();
- mb.sliceOperations(i*blen, Math.min((i+1)*blen, lrlen)-1,
- 0, mb.getNumColumns()-1, tmp);
- partBlocks[i] = tmp;
- }
-
- return partBlocks;
- }
-
- public static MatrixBlock[] partitionIntoColumnBlocks( MatrixBlock mb, int blen )
- throws DMLRuntimeException
- {
- //in-memory colblock partitioning (according to brlen of rdd)
- int lclen = mb.getNumColumns();
- int numBlocks = (int)Math.ceil((double)lclen/blen);
- MatrixBlock[] partBlocks = new MatrixBlock[numBlocks];
- for( int i=0; i<numBlocks; i++ )
- {
- MatrixBlock tmp = new MatrixBlock();
- mb.sliceOperations(0, mb.getNumRows()-1,
- i*blen, Math.min((i+1)*blen, lclen)-1, tmp);
- partBlocks[i] = tmp;
- }
-
- return partBlocks;
- }
-
// This returns RDD with identifier as well as location
public static String getStartLineFromSparkDebugInfo(String line) throws DMLRuntimeException {
// To remove: (2) -- Assumption: At max, 9 RDDs as input to transformation/action
@@ -211,85 +169,6 @@ public class SparkUtils
else
return retVal + "|" + twoSpaces;
}
-
-
- // len = {clen or rlen}, blen = {brlen or bclen}
- public static long getStartGlobalIndex(long blockIndex, int blen, long len) {
- return UtilFunctions.computeCellIndex(blockIndex, blen, 0);
- }
-
- public static JavaPairRDD<MatrixIndexes, MatrixBlock> getRDDWithEmptyBlocks(JavaSparkContext sc,
- JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocksWithoutEmptyBlocks,
- long numRows, long numColumns, int brlen, int bclen) throws DMLRuntimeException {
- JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocksWithEmptyBlocks = null;
- // ----------------------------------------------------------------------------
- // Now take care of empty blocks
- // This is done as non-rdd operation due to complexity involved in "not in" operations
- // Since this deals only with keys and not blocks, it might not be that bad.
- List<MatrixIndexes> indexes = binaryBlocksWithoutEmptyBlocks.keys().collect();
- ArrayList<Tuple2<MatrixIndexes, MatrixBlock> > emptyBlocksList = getEmptyBlocks(indexes, numRows, numColumns, brlen, bclen);
- if(emptyBlocksList != null && emptyBlocksList.size() > 0) {
- // Empty blocks needs to be inserted
- binaryBlocksWithEmptyBlocks = JavaPairRDD.fromJavaRDD(sc.parallelize(emptyBlocksList))
- .union(binaryBlocksWithoutEmptyBlocks);
- }
- else {
- binaryBlocksWithEmptyBlocks = binaryBlocksWithoutEmptyBlocks;
- }
- // ----------------------------------------------------------------------------
- return binaryBlocksWithEmptyBlocks;
- }
-
- private static ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> getEmptyBlocks(List<MatrixIndexes> nonEmptyIndexes, long rlen, long clen, int brlen, int bclen) throws DMLRuntimeException {
- long numBlocksPerRow = (long) Math.ceil((double)rlen / brlen);
- long numBlocksPerCol = (long) Math.ceil((double)clen / bclen);
- long expectedNumBlocks = numBlocksPerRow*numBlocksPerCol;
-
- if(expectedNumBlocks == nonEmptyIndexes.size()) {
- return null; // no empty blocks required: sanity check
- }
- else if(expectedNumBlocks < nonEmptyIndexes.size()) {
- throw new DMLRuntimeException("Error: Incorrect number of indexes in ReblockSPInstruction:" + nonEmptyIndexes.size());
- }
-
- // ----------------------------------------------------------------------------
- // Add empty blocks: Performs a "not-in" operation
- Collections.sort(nonEmptyIndexes); // sort in ascending order first wrt rows and then wrt columns
- ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> retVal = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>();
- int index = 0;
- for(long row = 1; row <= Math.ceil((double)rlen / brlen); row++) {
- for(long col = 1; col <= Math.ceil((double)clen / bclen); col++) {
- boolean matrixBlockExists = false;
- if(nonEmptyIndexes.size() > index) {
- matrixBlockExists = (nonEmptyIndexes.get(index).getRowIndex() == row) && (nonEmptyIndexes.get(index).getColumnIndex() == col);
- }
- if(matrixBlockExists) {
- index++; // No need to add empty block
- }
- else {
- // ------------------------------------------------------------------
- // Compute local block size:
- // Example: For matrix: 1500 X 1100 with block length 1000 X 1000
- // We will have four local block sizes (1000X1000, 1000X100, 500X1000 and 500X1000)
- long blockRowIndex = row;
- long blockColIndex = col;
- int emptyBlk_lrlen = UtilFunctions.computeBlockSize(rlen, blockRowIndex, brlen);
- int emptyBlk_lclen = UtilFunctions.computeBlockSize(clen, blockColIndex, bclen);
- // ------------------------------------------------------------------
-
- MatrixBlock emptyBlk = new MatrixBlock(emptyBlk_lrlen, emptyBlk_lclen, true);
- retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(blockRowIndex, blockColIndex), emptyBlk));
- }
- }
- }
- // ----------------------------------------------------------------------------
-
- if(index != nonEmptyIndexes.size()) {
- throw new DMLRuntimeException("Unexpected error while adding empty blocks");
- }
-
- return retVal;
- }
public static JavaPairRDD<MatrixIndexes, MatrixBlock> getEmptyBlockRDD( JavaSparkContext sc, MatrixCharacteristics mc )
{
@@ -323,18 +202,6 @@ public class SparkUtils
return ret;
}
- public static JavaPairRDD<MatrixIndexes, MatrixBlock> cacheBinaryBlockRDD(JavaPairRDD<MatrixIndexes, MatrixBlock> input)
- {
- JavaPairRDD<MatrixIndexes, MatrixBlock> ret = null;
-
- if( !input.getStorageLevel().equals(DEFAULT_TMP) ) {
- ret = SparkUtils.copyBinaryBlockMatrix(input, false)
- .persist(DEFAULT_TMP);
- }
-
- return ret;
- }
-
/**
* Utility to compute dimensions and non-zeros in a given RDD of binary cells.
*
@@ -350,24 +217,6 @@ public class SparkUtils
return ret;
}
-
- /**
- * Utility to compute dimensions and non-zeros in the given RDD of matrix blocks.
- *
- * @param input matrix as {@code JavaPairRDD<MatrixIndexes, MatrixBlock>}
- * @param brlen number of rows in a block
- * @param bclen number of columns in a block
- * @return matrix characteristics
- */
- public static MatrixCharacteristics computeMatrixCharacteristics(JavaPairRDD<MatrixIndexes, MatrixBlock> input, int brlen, int bclen)
- {
- // compute dimensions and nnz in single pass
- MatrixCharacteristics ret = input
- .map(new AnalyzeBlockMatrixCharacteristics(brlen, bclen))
- .reduce(new AggregateMatrixCharacteristics());
-
- return ret;
- }
private static class AnalyzeCellMatrixCharacteristics implements Function<Tuple2<MatrixIndexes,MatrixCell>, MatrixCharacteristics>
{
@@ -384,30 +233,6 @@ public class SparkUtils
}
}
- private static class AnalyzeBlockMatrixCharacteristics implements Function<Tuple2<MatrixIndexes,MatrixBlock>, MatrixCharacteristics>
- {
- private static final long serialVersionUID = -1857049501217936951L;
-
- private int _brlen = -1;
- private int _bclen = -1;
-
- public AnalyzeBlockMatrixCharacteristics(int brlen, int bclen) {
- _brlen = brlen;
- _bclen = bclen;
- }
-
- @Override
- public MatrixCharacteristics call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
- throws Exception
- {
- MatrixBlock block = arg0._2();
- long rlen = (arg0._1().getRowIndex()-1)*_brlen + block.getNumRows();
- long clen = (arg0._1().getColumnIndex()-1)*_bclen + block.getNumColumns();
- long nnz = block.getNonZeros();
- return new MatrixCharacteristics(rlen, clen, _brlen, _bclen, nnz);
- }
- }
-
private static class AggregateMatrixCharacteristics implements Function2<MatrixCharacteristics, MatrixCharacteristics, MatrixCharacteristics>
{
private static final long serialVersionUID = 4263886749699779994L;
@@ -424,28 +249,7 @@ public class SparkUtils
arg0.getNonZeros() + arg1.getNonZeros() ); //sum
}
}
-
- ////////////////////////////
- //TODO MB: to be cleaned up but still used
-
- /**
- * Utility to compute number of non-zeros from the given RDD of MatrixCells
- *
- * @param rdd matrix as {@code JavaPairRDD<MatrixIndexes, MatrixCell>}
- * @return number of non-zeros
- */
- public static long computeNNZFromCells(JavaPairRDD<MatrixIndexes, MatrixCell> rdd) {
- long nnz = rdd.values().filter(
- new Function<MatrixCell,Boolean>() {
- private static final long serialVersionUID = -6550193680630537857L;
- @Override
- public Boolean call(MatrixCell v1) throws Exception {
- return (v1.getValue() != 0);
- }
- }).count();
- return nnz;
- }
-
+
/**
* Utility to compute number of non-zeros from the given RDD of MatrixBlocks
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
index afd1b60..6300b32 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
@@ -38,20 +38,6 @@ public class FrameReaderFactory
return createFrameReader(iinfo, props);
}
- public static FrameReader createFrameReader( ReadProperties rprops )
- throws DMLRuntimeException
- {
- //check valid read properties
- if( rprops == null )
- throw new DMLRuntimeException("Failed to create frame reader with empty properties.");
-
- InputInfo iinfo = rprops.inputInfo;
- FileFormatProperties props = (iinfo==InputInfo.CSVInputInfo) ? ((rprops.formatProperties!=null) ?
- (CSVFileFormatProperties)rprops.formatProperties : new CSVFileFormatProperties()) : null;
-
- return createFrameReader(iinfo, props);
- }
-
public static FrameReader createFrameReader( InputInfo iinfo, FileFormatProperties props )
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
index f3533af..479ce40 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
@@ -25,7 +25,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
@@ -238,91 +237,6 @@ public class WriterTextCSV extends MatrixWriter
}
}
-
-
- /**
- * Method to merge multiple CSV part files on HDFS into a single CSV file on HDFS.
- * The part files are created by CSV_WRITE MR job.
- *
- * This method is invoked from CP-write instruction.
- *
- * @param srcFileName source file name
- * @param destFileName destination file name
- * @param csvprop CSV file format properties
- * @param rlen number of rows
- * @param clen number of columns
- * @throws IOException if IOException occurs
- */
- public final void mergeCSVPartFiles(String srcFileName, String destFileName, CSVFileFormatProperties csvprop, long rlen, long clen)
- throws IOException
- {
- Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf());
-
- Path srcFilePath = new Path(srcFileName);
- Path mergedFilePath = new Path(destFileName);
- FileSystem hdfs = FileSystem.get(conf);
-
- if (hdfs.exists(mergedFilePath)) {
- hdfs.delete(mergedFilePath, true);
- }
- OutputStream out = hdfs.create(mergedFilePath, true);
-
- // write out the header, if needed
- if (csvprop.hasHeader()) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < clen; i++) {
- sb.append("C" + (i + 1));
- if (i < clen - 1)
- sb.append(csvprop.getDelim());
- }
- sb.append('\n');
- out.write(sb.toString().getBytes());
- sb.setLength(0);
- }
-
- // if the source is a directory
- if (hdfs.isDirectory(srcFilePath)) {
- try {
- FileStatus[] contents = hdfs.listStatus(srcFilePath);
- Path[] partPaths = new Path[contents.length];
- int numPartFiles = 0;
- for (int i = 0; i < contents.length; i++) {
- if (!contents[i].isDirectory()) {
- partPaths[i] = contents[i].getPath();
- numPartFiles++;
- }
- }
- Arrays.sort(partPaths);
-
- for (int i = 0; i < numPartFiles; i++) {
- InputStream in = hdfs.open(partPaths[i]);
- try {
- IOUtils.copyBytes(in, out, conf, false);
- if(i<numPartFiles-1)
- out.write('\n');
- }
- finally {
- IOUtilFunctions.closeSilently(in);
- }
- }
- } finally {
- IOUtilFunctions.closeSilently(out);
- }
- } else if (hdfs.isFile(srcFilePath)) {
- InputStream in = null;
- try {
- in = hdfs.open(srcFilePath);
- IOUtils.copyBytes(in, out, conf, true);
- } finally {
- IOUtilFunctions.closeSilently(in);
- IOUtilFunctions.closeSilently(out);
- }
- } else {
- throw new IOException(srcFilePath.toString()
- + ": No such file or directory");
- }
- }
-
@SuppressWarnings("unchecked")
public final void addHeaderToCSV(String srcFileName, String destFileName, long rlen, long clen)
throws IOException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1ab2d7e6/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
index 5638dad..9c6af97 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
@@ -508,22 +508,7 @@ public class LibMatrixReorg
return out;
}
-
- /**
- * CP rmempty operation (single input, single output matrix)
- *
- * @param in input matrix
- * @param ret output matrix
- * @param rows ?
- * @return matrix block
- * @throws DMLRuntimeException if DMLRuntimeException occurs
- */
- public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows)
- throws DMLRuntimeException
- {
- return rmempty(in, ret, rows, null);
- }
-
+
/**
* CP rmempty operation (single input, single output matrix)
*