You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2016/11/14 23:28:36 UTC
[3/5] incubator-systemml git commit: [SYSTEMML-842] Javadoc cleanup
in runtime compress and controlprogram packages
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index 27655bf..9b2d650 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -124,7 +124,7 @@ public class SparkExecutionContext extends ExecutionContext
* Returns the used singleton spark context. In case of lazy spark context
* creation, this methods blocks until the spark context is created.
*
- * @return
+ * @return java spark context
*/
public JavaSparkContext getSparkContext()
{
@@ -137,11 +137,7 @@ public class SparkExecutionContext extends ExecutionContext
//return the created spark context
return _spctx;
}
-
- /**
- *
- * @return
- */
+
public static JavaSparkContext getSparkContextStatic() {
initSparkContext();
return _spctx;
@@ -151,22 +147,16 @@ public class SparkExecutionContext extends ExecutionContext
* Indicates if the spark context has been created or has
* been passed in from outside.
*
- * @return
+ * @return true if spark context created
*/
public synchronized static boolean isSparkContextCreated() {
return (_spctx != null);
}
-
- /**
- *
- */
+
public static void resetSparkContextStatic() {
_spctx = null;
}
-
- /**
- *
- */
+
public void close()
{
synchronized( SparkExecutionContext.class ) {
@@ -185,10 +175,7 @@ public class SparkExecutionContext extends ExecutionContext
public static boolean isLazySparkContextCreation(){
return LAZY_SPARKCTX_CREATION;
}
-
- /**
- *
- */
+
private synchronized static void initSparkContext()
{
//check for redundant spark context init
@@ -255,7 +242,7 @@ public class SparkExecutionContext extends ExecutionContext
* Sets up a SystemML-preferred Spark configuration based on the implicit
* default configuration (as passed via configurations from outside).
*
- * @return
+ * @return spark configuration
*/
public static SparkConf createSystemMLSparkConf() {
SparkConf conf = new SparkConf();
@@ -282,9 +269,9 @@ public class SparkExecutionContext extends ExecutionContext
* Spark instructions should call this for all matrix inputs except broadcast
* variables.
*
- * @param varname
- * @return
- * @throws DMLRuntimeException
+ * @param varname varible name
+ * @return JavaPairRDD of MatrixIndexes-MatrixBlocks
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
@SuppressWarnings("unchecked")
public JavaPairRDD<MatrixIndexes,MatrixBlock> getBinaryBlockRDDHandleForVariable( String varname )
@@ -297,9 +284,9 @@ public class SparkExecutionContext extends ExecutionContext
* Spark instructions should call this for all frame inputs except broadcast
* variables.
*
- * @param varname
- * @return
- * @throws DMLRuntimeException
+ * @param varname variable name
+ * @return JavaPairRDD of Longs-FrameBlocks
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
@SuppressWarnings("unchecked")
public JavaPairRDD<Long,FrameBlock> getFrameBinaryBlockRDDHandleForVariable( String varname )
@@ -308,14 +295,7 @@ public class SparkExecutionContext extends ExecutionContext
JavaPairRDD<Long,FrameBlock> out = (JavaPairRDD<Long,FrameBlock>) getRDDHandleForVariable( varname, InputInfo.BinaryBlockInputInfo);
return out;
}
-
- /**
- *
- * @param varname
- * @param inputInfo
- * @return
- * @throws DMLRuntimeException
- */
+
public JavaPairRDD<?,?> getRDDHandleForVariable( String varname, InputInfo inputInfo )
throws DMLRuntimeException
{
@@ -337,10 +317,10 @@ public class SparkExecutionContext extends ExecutionContext
* This call returns an RDD handle for a given matrix object. This includes
* the creation of RDDs for in-memory or binary-block HDFS data.
*
- *
- * @param varname
- * @return
- * @throws DMLRuntimeException
+ * @param mo matrix object
+ * @param inputInfo input info
+ * @return JavaPairRDD handle for a matrix object
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
@SuppressWarnings("unchecked")
public JavaPairRDD<?,?> getRDDHandleForMatrixObject( MatrixObject mo, InputInfo inputInfo )
@@ -422,10 +402,10 @@ public class SparkExecutionContext extends ExecutionContext
* FIXME: currently this implementation assumes matrix representations but frame signature
* in order to support the old transform implementation.
*
- * @param mo
- * @param inputInfo
- * @return
- * @throws DMLRuntimeException
+ * @param fo frame object
+ * @param inputInfo input info
+ * @return JavaPairRDD handle for a frame object
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
@SuppressWarnings("unchecked")
public JavaPairRDD<?,?> getRDDHandleForFrameObject( FrameObject fo, InputInfo inputInfo )
@@ -511,9 +491,9 @@ public class SparkExecutionContext extends ExecutionContext
* However, in order to handle this, we need to keep track when broadcast
* variables are no longer required.
*
- * @param varname
- * @return
- * @throws DMLRuntimeException
+ * @param varname variable name
+ * @return wrapper for broadcast variables
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
@SuppressWarnings("unchecked")
public PartitionedBroadcast<MatrixBlock> getBroadcastForVariable( String varname )
@@ -575,14 +555,6 @@ public class SparkExecutionContext extends ExecutionContext
return bret;
}
-
- /**
- *
- * @param varname
- * @return
- * @throws DMLRuntimeException
- */
-
@SuppressWarnings("unchecked")
public PartitionedBroadcast<FrameBlock> getBroadcastForFrameVariable( String varname)
throws DMLRuntimeException
@@ -642,15 +614,7 @@ public class SparkExecutionContext extends ExecutionContext
return bret;
}
-
-
- /**
- *
- * @param varname
- * @return
- * @throws DMLRuntimeException
- */
public BlockPartitioner getPartitionerForRDDVariable(String varname)
throws DMLRuntimeException
{
@@ -666,9 +630,9 @@ public class SparkExecutionContext extends ExecutionContext
* Keep the output rdd of spark rdd operations as meta data of matrix/frame
* objects in the symbol table.
*
- * @param varname
- * @param rdd
- * @throws DMLRuntimeException
+ * @param varname variable name
+ * @param rdd JavaPairRDD handle for variable
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public void setRDDHandleForVariable(String varname, JavaPairRDD<?,?> rdd)
throws DMLRuntimeException
@@ -681,10 +645,12 @@ public class SparkExecutionContext extends ExecutionContext
/**
* Utility method for creating an RDD out of an in-memory matrix block.
*
- * @param sc
- * @param block
- * @return
- * @throws DMLRuntimeException
+ * @param sc java spark context
+ * @param src matrix block
+ * @param brlen block row length
+ * @param bclen block column length
+ * @return JavaPairRDD handle to matrix block
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static JavaPairRDD<MatrixIndexes,MatrixBlock> toMatrixJavaPairRDD(JavaSparkContext sc, MatrixBlock src, int brlen, int bclen)
throws DMLRuntimeException
@@ -731,14 +697,7 @@ public class SparkExecutionContext extends ExecutionContext
return result;
}
-
- /**
- *
- * @param sc
- * @param src
- * @return
- * @throws DMLRuntimeException
- */
+
public static JavaPairRDD<Long,FrameBlock> toFrameJavaPairRDD(JavaSparkContext sc, FrameBlock src)
throws DMLRuntimeException
{
@@ -774,13 +733,15 @@ public class SparkExecutionContext extends ExecutionContext
/**
* This method is a generic abstraction for calls from the buffer pool.
- * See toMatrixBlock(JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, int numRows, int numCols);
*
- * @param rdd
- * @param numRows
- * @param numCols
- * @return
- * @throws DMLRuntimeException
+ * @param rdd rdd object
+ * @param rlen number of rows
+ * @param clen number of columns
+ * @param brlen number of rows in a block
+ * @param bclen number of columns in a block
+ * @param nnz number of non-zeros
+ * @return matrix block
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
@SuppressWarnings("unchecked")
public static MatrixBlock toMatrixBlock(RDDObject rdd, int rlen, int clen, int brlen, int bclen, long nnz)
@@ -798,11 +759,14 @@ public class SparkExecutionContext extends ExecutionContext
* NOTE: This is an unguarded utility function, which requires memory for both the output matrix
* and its collected, blocked representation.
*
- * @param rdd
- * @param numRows
- * @param numCols
- * @return
- * @throws DMLRuntimeException
+ * @param rdd JavaPairRDD for matrix block
+ * @param rlen number of rows
+ * @param clen number of columns
+ * @param brlen number of rows in a block
+ * @param bclen number of columns in a block
+ * @param nnz number of non-zeros
+ * @return matrix block
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static MatrixBlock toMatrixBlock(JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, int rlen, int clen, int brlen, int bclen, long nnz)
throws DMLRuntimeException
@@ -887,12 +851,12 @@ public class SparkExecutionContext extends ExecutionContext
* Utility method for creating a single matrix block out of a binary cell RDD.
* Note that this collect call might trigger execution of any pending transformations.
*
- * @param rdd
- * @param rlen
- * @param clen
- * @param nnz
- * @return
- * @throws DMLRuntimeException
+ * @param rdd JavaPairRDD for matrix block
+ * @param rlen number of rows
+ * @param clen number of columns
+ * @param nnz number of non-zeros
+ * @return matrix block
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static MatrixBlock toMatrixBlock(JavaPairRDD<MatrixIndexes, MatrixCell> rdd, int rlen, int clen, long nnz)
throws DMLRuntimeException
@@ -935,18 +899,7 @@ public class SparkExecutionContext extends ExecutionContext
return out;
}
-
- /**
- *
- * @param rdd
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param nnz
- * @return
- * @throws DMLRuntimeException
- */
+
public static PartitionedBlock<MatrixBlock> toPartitionedMatrixBlock(JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, int rlen, int clen, int brlen, int bclen, long nnz)
throws DMLRuntimeException
{
@@ -973,15 +926,6 @@ public class SparkExecutionContext extends ExecutionContext
return out;
}
- /**
- *
- * @param rdd
- * @param schema
- * @param rlen
- * @param clen
- * @return
- * @throws DMLRuntimeException
- */
@SuppressWarnings("unchecked")
public static FrameBlock toFrameBlock(RDDObject rdd, ValueType[] schema, int rlen, int clen)
throws DMLRuntimeException
@@ -989,16 +933,7 @@ public class SparkExecutionContext extends ExecutionContext
JavaPairRDD<Long,FrameBlock> lrdd = (JavaPairRDD<Long,FrameBlock>) rdd.getRDD();
return toFrameBlock(lrdd, schema, rlen, clen);
}
-
- /**
- *
- * @param rdd
- * @param schema
- * @param rlen
- * @param clen
- * @return
- * @throws DMLRuntimeException
- */
+
public static FrameBlock toFrameBlock(JavaPairRDD<Long,FrameBlock> rdd, ValueType[] schema, int rlen, int clen)
throws DMLRuntimeException
{
@@ -1035,12 +970,7 @@ public class SparkExecutionContext extends ExecutionContext
return out;
}
-
- /**
- *
- * @param rdd
- * @param oinfo
- */
+
@SuppressWarnings("unchecked")
public static long writeRDDtoHDFS( RDDObject rdd, String path, OutputInfo oinfo )
{
@@ -1058,12 +988,7 @@ public class SparkExecutionContext extends ExecutionContext
//return nnz aggregate of all blocks
return nnz;
}
-
- /**
- *
- * @param rdd
- * @param oinfo
- */
+
@SuppressWarnings("unchecked")
public static void writeFrameRDDtoHDFS( RDDObject rdd, String path, OutputInfo oinfo )
{
@@ -1090,9 +1015,9 @@ public class SparkExecutionContext extends ExecutionContext
/**
* Adds a child rdd object to the lineage of a parent rdd.
*
- * @param varParent
- * @param varChild
- * @throws DMLRuntimeException
+ * @param varParent parent variable
+ * @param varChild child variable
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public void addLineageRDD(String varParent, String varChild)
throws DMLRuntimeException
@@ -1106,9 +1031,9 @@ public class SparkExecutionContext extends ExecutionContext
/**
* Adds a child broadcast object to the lineage of a parent rdd.
*
- * @param varParent
- * @param varChild
- * @throws DMLRuntimeException
+ * @param varParent parent variable
+ * @param varChild child variable
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public void addLineageBroadcast(String varParent, String varChild)
throws DMLRuntimeException
@@ -1118,14 +1043,7 @@ public class SparkExecutionContext extends ExecutionContext
parent.addLineageChild( child );
}
-
- /**
- *
- * @param varParent
- * @param varChild
- * @param broadcast
- * @throws DMLRuntimeException
- */
+
public void addLineage(String varParent, String varChild, boolean broadcast)
throws DMLRuntimeException
{
@@ -1180,12 +1098,7 @@ public class SparkExecutionContext extends ExecutionContext
throw new DMLRuntimeException(ex);
}
}
-
- /**
- *
- * @param lob
- * @throws IOException
- */
+
@SuppressWarnings({ "rawtypes", "unchecked" })
private void rCleanupLineageObject(LineageObject lob)
throws IOException
@@ -1227,8 +1140,7 @@ public class SparkExecutionContext extends ExecutionContext
* Hence, it is intended to be used on rmvar only. Depending on the
* ASYNCHRONOUS_VAR_DESTROY configuration, this is asynchronous or not.
*
- *
- * @param inV
+ * @param bvar broadcast variable
*/
public void cleanupBroadcastVariable(Broadcast<?> bvar)
{
@@ -1244,7 +1156,7 @@ public class SparkExecutionContext extends ExecutionContext
* Hence, it is intended to be used on rmvar only. Depending on the
* ASYNCHRONOUS_VAR_DESTROY configuration, this is asynchronous or not.
*
- * @param rvar
+ * @param rvar rdd variable to remove
*/
public void cleanupRDDVariable(JavaPairRDD<?,?> rvar)
{
@@ -1252,12 +1164,7 @@ public class SparkExecutionContext extends ExecutionContext
rvar.unpersist( ASYNCHRONOUS_VAR_DESTROY );
}
}
-
- /**
- *
- * @param var
- * @throws DMLRuntimeException
- */
+
@SuppressWarnings("unchecked")
public void repartitionAndCacheMatrixObject( String var )
throws DMLRuntimeException
@@ -1305,12 +1212,7 @@ public class SparkExecutionContext extends ExecutionContext
outro.addLineageChild(inro); //keep lineage to prevent cycles on cleanup
mo.setRDDHandle(outro);
}
-
- /**
- *
- * @param var
- * @throws DMLRuntimeException
- */
+
@SuppressWarnings("unchecked")
public void cacheMatrixObject( String var )
throws DMLRuntimeException
@@ -1330,43 +1232,26 @@ public class SparkExecutionContext extends ExecutionContext
if( !isRDDCached(in.id()) )
in.count(); //trigger caching to prevent contention
}
-
- /**
- *
- * @param poolName
- */
+
public void setThreadLocalSchedulerPool(String poolName) {
if( FAIR_SCHEDULER_MODE ) {
getSparkContext().sc().setLocalProperty(
"spark.scheduler.pool", poolName);
}
}
-
- /**
- *
- */
+
public void cleanupThreadLocalSchedulerPool() {
if( FAIR_SCHEDULER_MODE ) {
getSparkContext().sc().setLocalProperty(
"spark.scheduler.pool", null);
}
}
-
- /**
- *
- * @param rddID
- * @return
- */
+
private boolean isRDDMarkedForCaching( int rddID ) {
JavaSparkContext jsc = getSparkContext();
return jsc.sc().getPersistentRDDs().contains(rddID);
}
-
- /**
- *
- * @param rddID
- * @return
- */
+
private boolean isRDDCached( int rddID ) {
//check that rdd is marked for caching
JavaSparkContext jsc = getSparkContext();
@@ -1387,12 +1272,6 @@ public class SparkExecutionContext extends ExecutionContext
// Debug String Handling (see explain); TODO to be removed
///////
- /**
- *
- * @param inst
- * @param outputVarName
- * @throws DMLRuntimeException
- */
public void setDebugString(SPInstruction inst, String outputVarName)
throws DMLRuntimeException
{
@@ -1513,7 +1392,7 @@ public class SparkExecutionContext extends ExecutionContext
/**
* Obtains the lazily analyzed spark cluster configuration.
*
- * @return
+ * @return spark cluster configuration
*/
public static SparkClusterConfig getSparkClusterConfig() {
//lazy creation of spark cluster config
@@ -1525,7 +1404,7 @@ public class SparkExecutionContext extends ExecutionContext
/**
* Obtains the available memory budget for broadcast variables in bytes.
*
- * @return
+ * @return broadcast memory budget
*/
public static double getBroadcastMemoryBudget() {
return getSparkClusterConfig()
@@ -1537,7 +1416,7 @@ public class SparkExecutionContext extends ExecutionContext
*
* @param min flag for minimum data budget
* @param refresh flag for refresh with spark context
- * @return
+ * @return data memory budget
*/
public static double getDataMemoryBudget(boolean min, boolean refresh) {
return getSparkClusterConfig()
@@ -1547,7 +1426,7 @@ public class SparkExecutionContext extends ExecutionContext
/**
* Obtain the number of executors in the cluster (excluding the driver).
*
- * @return
+ * @return number of executors
*/
public static int getNumExecutors() {
return getSparkClusterConfig()
@@ -1558,16 +1437,13 @@ public class SparkExecutionContext extends ExecutionContext
* Obtain the default degree of parallelism (cores in the cluster).
*
* @param refresh flag for refresh with spark context
- * @return
+ * @return default degree of parallelism
*/
public static int getDefaultParallelism(boolean refresh) {
return getSparkClusterConfig()
.getDefaultParallelism(refresh);
}
-
- /**
- *
- */
+
public void checkAndRaiseValidationWarningJDKVersion()
{
//check for jdk version less than jdk8
@@ -1643,21 +1519,11 @@ public class SparkExecutionContext extends ExecutionContext
if( LOG.isDebugEnabled() )
LOG.debug( this.toString() );
}
-
- /**
- *
- * @return
- */
+
public long getBroadcastMemoryBudget() {
return (long) (_memExecutor * _memBroadcastFrac);
}
-
- /**
- *
- * @param min
- * @param refresh
- * @return
- */
+
public long getDataMemoryBudget(boolean min, boolean refresh) {
//always get the current num executors on refresh because this might
//change if not all executors are initially allocated and it is plan-relevant
@@ -1672,21 +1538,12 @@ public class SparkExecutionContext extends ExecutionContext
(min ? _memDataMinFrac : _memDataMaxFrac) );
}
- /**
- *
- * @return
- */
public int getNumExecutors() {
if( _numExecutors < 0 )
analyzeSparkParallelismConfiguation(null);
return _numExecutors;
}
-
- /**
- *
- * @param refresh
- * @return
- */
+
public int getDefaultParallelism(boolean refresh) {
if( _defaultPar < 0 && !refresh )
analyzeSparkParallelismConfiguation(null);
@@ -1697,10 +1554,6 @@ public class SparkExecutionContext extends ExecutionContext
getSparkContextStatic().defaultParallelism() : _defaultPar;
}
- /**
- *
- * @param conf
- */
public void analyzeSparkConfiguationLegacy(SparkConf conf) {
//ensure allocated spark conf
SparkConf sconf = (conf == null) ? createSystemMLSparkConf() : conf;
@@ -1718,11 +1571,7 @@ public class SparkExecutionContext extends ExecutionContext
//analyze spark degree of parallelism
analyzeSparkParallelismConfiguation(sconf);
}
-
- /**
- *
- * @param conf
- */
+
public void analyzeSparkConfiguation(SparkConf conf) {
//ensure allocated spark conf
SparkConf sconf = (conf == null) ? createSystemMLSparkConf() : conf;
@@ -1740,11 +1589,7 @@ public class SparkExecutionContext extends ExecutionContext
//analyze spark degree of parallelism
analyzeSparkParallelismConfiguation(sconf);
}
-
- /**
- *
- * @param sconf
- */
+
private void analyzeSparkParallelismConfiguation(SparkConf sconf) {
int numExecutors = sconf.getInt("spark.executor.instances", -1);
int numCoresPerExec = sconf.getInt("spark.executor.cores", -1);
@@ -1771,7 +1616,7 @@ public class SparkExecutionContext extends ExecutionContext
* we simply get it from the context; otherwise, we use Spark internal
* constants to avoid creating the spark context just for the version.
*
- * @return
+ * @return spark version string
*/
private String getSparkVersionString() {
//check for existing spark context
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
index ca49493..ab76b55 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
@@ -62,30 +62,13 @@ public abstract class DataPartitioner
_format = dpf;
_n = n;
}
-
-
-
- /**
- *
- * @param in
- * @param fnameNew
- * @return
- * @throws DMLRuntimeException
- */
+
public MatrixObject createPartitionedMatrixObject( MatrixObject in, String fnameNew )
throws DMLRuntimeException
{
return createPartitionedMatrixObject(in, fnameNew, false);
}
-
- /**
- *
- * @param in
- * @param fnameNew
- * @param force
- * @return
- * @throws DMLRuntimeException
- */
+
public MatrixObject createPartitionedMatrixObject( MatrixObject in, String fnameNew, boolean force )
throws DMLRuntimeException
{
@@ -106,10 +89,11 @@ public abstract class DataPartitioner
* created matrix object can be used transparently for obtaining the full matrix
* or reading 1 or multiple partitions based on given index ranges.
*
- * @param in
- * @param force
- * @return
- * @throws DMLRuntimeException
+ * @param in input matrix object
+ * @param out output matrix object
+ * @param force if false, try to optimize
+ * @return partitioned matrix object
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public MatrixObject createPartitionedMatrixObject( MatrixObject in, MatrixObject out, boolean force )
throws DMLRuntimeException
@@ -190,27 +174,12 @@ public abstract class DataPartitioner
return out;
}
-
- /**
- *
- */
+
public void disableBinaryCell()
{
_allowBinarycell = false;
}
-
- /**
- *
- * @param fname
- * @param fnameNew
- * @param ii
- * @param oi
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
+
protected abstract void partitionMatrix( MatrixObject in, String fnameNew, InputInfo ii, OutputInfo oi, long rlen, long clen, int brlen, int bclen )
throws DMLRuntimeException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
index 72cf976..8fcef65 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
@@ -86,11 +86,12 @@ public class DataPartitionerLocal extends DataPartitioner
private int _par = -1;
/**
+ * DataPartitionerLocal constructor.
*
- * @param dpf
- * @param n
+ * @param dpf data partitionformat
+ * @param n ?
* @param par -1 for serial otherwise number of threads, can be ignored by implementation
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public DataPartitionerLocal(PDataPartitionFormat dpf, int n, int par)
throws DMLRuntimeException
@@ -133,18 +134,6 @@ public class DataPartitionerLocal extends DataPartitioner
LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
}
-
-
-
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
private void partitionTextCell( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
throws DMLRuntimeException
{
@@ -239,15 +228,6 @@ public class DataPartitionerLocal extends DataPartitioner
}
}
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
@SuppressWarnings("deprecation")
private void partitionBinaryCell( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
throws DMLRuntimeException
@@ -337,16 +317,7 @@ public class DataPartitionerLocal extends DataPartitioner
throw new DMLRuntimeException("Unable to partition binary cell matrix.", e);
}
}
-
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
+
@SuppressWarnings("deprecation")
private void partitionBinaryBlock( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
throws DMLRuntimeException
@@ -425,15 +396,6 @@ public class DataPartitionerLocal extends DataPartitioner
}
}
- /**
- *
- * @param fname
- * @param fnameStaging
- * @param fnameNew
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- */
@SuppressWarnings("deprecation")
private void partitionBinaryBlock2BinaryCell( String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen )
throws DMLRuntimeException
@@ -541,18 +503,6 @@ public class DataPartitionerLocal extends DataPartitioner
}
}
-
- /**
- *
- * @param dir
- * @param mb
- * @param row_offset
- * @param col_offset
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- * @throws IOException
- */
private void appendBlockToStagingArea( String dir, MatrixBlock mb, long row_offset, long col_offset, long brlen, long bclen )
throws DMLRuntimeException, IOException
{
@@ -602,16 +552,7 @@ public class DataPartitionerLocal extends DataPartitioner
LocalFileUtils.writeMatrixBlockToLocal(pfname, mb);
}
}
-
- /**
- *
- * @param dir
- * @param buffer
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- * @throws IOException
- */
+
private void appendCellBufferToStagingArea( String dir, LinkedList<Cell> buffer, int brlen, int bclen )
throws DMLRuntimeException, IOException
{
@@ -775,24 +716,14 @@ public class DataPartitionerLocal extends DataPartitioner
// Helper methods for local fs //
// read/write //
/////////////////////////////////
-
- /**
- *
- * @param dir
- * @return
- */
+
private long getKeyFromFilePath( String dir )
{
String[] dirparts = dir.split("/");
long key = Long.parseLong( dirparts[dirparts.length-1] );
return key;
}
-
- /**
- *
- * @param fname
- * @return
- */
+
private long getKey2FromFileName( String fname )
{
return Long.parseLong( fname.split("_")[1] );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
index a339915..6fcfc1a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
@@ -56,10 +56,7 @@ public class DataPartitionerRemoteMapper
{
}
-
- /**
- *
- */
+
@Override
public void map(Writable key, Writable value, OutputCollector<Writable, Writable> out, Reporter reporter)
throws IOException
@@ -67,9 +64,6 @@ public class DataPartitionerRemoteMapper
_mapper.processKeyValue(key, value, out, reporter);
}
- /**
- *
- */
@Override
public void configure(JobConf job)
{
@@ -102,10 +96,7 @@ public class DataPartitionerRemoteMapper
else
throw new RuntimeException("Unable to configure mapper with unknown input info: "+ii.toString());
}
-
- /**
- *
- */
+
@Override
public void close()
throws IOException
@@ -407,10 +398,7 @@ public class DataPartitionerRemoteMapper
}
}
}
-
- /**
- *
- */
+
private class DataPartitionerMapperBinaryblock2Binarycell extends DataPartitionerMapper
{
private JobConf _cachedJobConf = null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
index 0293299..8a9ddce 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteReducer.java
@@ -69,9 +69,6 @@ public class DataPartitionerRemoteReducer
_reducer.processKeyValueList(key, valueList, out, reporter);
}
- /**
- *
- */
public void configure(JobConf job)
{
String fnameNew = MRJobConfiguration.getPartitioningFilename( job );
@@ -86,10 +83,7 @@ public class DataPartitionerRemoteReducer
else
throw new RuntimeException("Unable to configure reducer with unknown output info: "+oi.toString());
}
-
- /**
- *
- */
+
@Override
public void close() throws IOException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
index 9736e69..41fb235 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSpark.java
@@ -96,16 +96,7 @@ public class DataPartitionerRemoteSpark extends DataPartitioner
Statistics.maintainCPHeavyHitters(jobname, System.nanoTime()-t0);
}
}
-
- /**
- *
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @param numRed
- * @return
- */
+
private long determineNumReducers(long rlen, long clen, int brlen, int bclen, long numRed)
{
//set the number of mappers and reducers
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
index 334094d..8caac98 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkReducer.java
@@ -38,9 +38,6 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo;
import scala.Tuple2;
-/**
- *
- */
public class DataPartitionerRemoteSparkReducer implements VoidFunction<Tuple2<Long, Iterable<Writable>>>
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
index 848ca7d..1ebaf00 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java
@@ -55,8 +55,8 @@ public class LocalTaskQueue<T>
/**
* Synchronized insert of a new task to the end of the FIFO queue.
*
- * @param t
- * @throws InterruptedException
+ * @param t task
+ * @throws InterruptedException if InterruptedException occurs
*/
public synchronized void enqueueTask( T t )
throws InterruptedException
@@ -75,8 +75,8 @@ public class LocalTaskQueue<T>
/**
* Synchronized read and delete from the top of the FIFO queue.
*
- * @return
- * @throws InterruptedException
+ * @return task
+ * @throws InterruptedException if InterruptedException occurs
*/
@SuppressWarnings("unchecked")
public synchronized T dequeueTask()
@@ -110,8 +110,8 @@ public class LocalTaskQueue<T>
/**
* Synchronized read of the current number of tasks in the queue.
*
- * @return
- * @throws InterruptedException
+ * @return number of tasks in queue
+ * @throws InterruptedException if InterruptedException occurs
*/
public synchronized int size()
throws InterruptedException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
index b18d5b8..5c5c5a6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ParWorker.java
@@ -79,11 +79,7 @@ public abstract class ParWorker
_numTasks = 0;
_numIters = 0;
}
-
- /**
- *
- * @return
- */
+
public LocalVariableMap getVariables()
{
return _ec.getVariables();
@@ -93,7 +89,7 @@ public abstract class ParWorker
* Returns a summary statistic of executed tasks and hence should only be called
* after execution.
*
- * @return
+ * @return number of executed tasks
*/
public long getExecutedTasks()
{
@@ -104,25 +100,19 @@ public abstract class ParWorker
* Returns a summary statistic of executed iterations and hence should only be called
* after execution.
*
- * @return
+ * @return number of executed iterations
*/
public long getExecutedIterations()
{
return _numIters;
}
-
- /**
- *
- */
+
public void resetExecutedTasks()
{
_numTasks = 0;
_numIters = 0;
}
-
- /**
- *
- */
+
protected void pinResultVariables()
{
for( String var : _resultVars )
@@ -136,11 +126,6 @@ public abstract class ParWorker
}
}
- /**
- *
- * @param task
- * @throws DMLRuntimeException
- */
protected void executeTask( Task task )
throws DMLRuntimeException
{
@@ -156,12 +141,7 @@ public abstract class ParWorker
break;
}
}
-
- /**
- *
- * @param task
- * @throws DMLRuntimeException
- */
+
private void executeSetTask( Task task )
throws DMLRuntimeException
{
@@ -202,12 +182,7 @@ public abstract class ParWorker
StatisticMonitor.putPWStat(_workerID, Stat.PARWRK_TASK_T, time2.stop());
}
}
-
- /**
- *
- * @param task
- * @throws DMLRuntimeException
- */
+
private void executeRangeTask( Task task )
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index adf0144..16d00b9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -158,8 +158,10 @@ public class ProgramConverter
* Creates a deep copy of the given execution context.
* For rt_platform=Hadoop, execution context has a symbol table.
*
- * @throws CloneNotSupportedException
- * @throws DMLRuntimeException
+ * @param ec execution context
+ * @return execution context
+ * @throws CloneNotSupportedException if CloneNotSupportedException occurs
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static ExecutionContext createDeepCopyExecutionContext(ExecutionContext ec)
throws CloneNotSupportedException, DMLRuntimeException
@@ -199,13 +201,15 @@ public class ProgramConverter
* specified parallel worker in order to avoid conflicts between parworkers. This happens recursively in order
* to support arbitrary control-flow constructs within a parfor.
*
- * @param childBlocks
- * @param pid
- * @param plain full deep copy without id replacement
- * @param forceDeepCopy
- *
- * @return
- * @throws DMLRuntimeException
+ * @param childBlocks child program blocks
+ * @param pid ?
+ * @param IDPrefix ?
+ * @param fnStack ?
+ * @param fnCreated ?
+ * @param plain if true, full deep copy without id replacement
+ * @param forceDeepCopy if true, force deep copy
+ * @return list of program blocks
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static ArrayList<ProgramBlock> rcreateDeepCopyProgramBlocks(ArrayList<ProgramBlock> childBlocks, long pid, int IDPrefix, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
throws DMLRuntimeException
@@ -258,16 +262,7 @@ public class ProgramConverter
return tmp;
}
-
- /**
- *
- * @param wpb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- */
+
public static WhileProgramBlock createDeepCopyWhileProgramBlock(WhileProgramBlock wpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
throws DMLRuntimeException
{
@@ -282,16 +277,7 @@ public class ProgramConverter
return tmpPB;
}
-
- /**
- *
- * @param ipb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- */
+
public static IfProgramBlock createDeepCopyIfProgramBlock(IfProgramBlock ipb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
throws DMLRuntimeException
{
@@ -307,16 +293,7 @@ public class ProgramConverter
return tmpPB;
}
-
- /**
- *
- * @param fpb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- */
+
public static ForProgramBlock createDeepCopyForProgramBlock(ForProgramBlock fpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
throws DMLRuntimeException
{
@@ -332,14 +309,7 @@ public class ProgramConverter
return tmpPB;
}
-
- /**
- *
- * @param fpb
- * @param prog
- * @return
- * @throws DMLRuntimeException
- */
+
public static ForProgramBlock createShallowCopyForProgramBlock(ForProgramBlock fpb, Program prog )
throws DMLRuntimeException
{
@@ -353,14 +323,7 @@ public class ProgramConverter
return tmpPB;
}
-
- /**
- *
- * @param pfpb
- * @param prog
- * @return
- * @throws DMLRuntimeException
- */
+
public static ForProgramBlock createShallowCopyParForProgramBlock(ParForProgramBlock pfpb, Program prog )
throws DMLRuntimeException
{
@@ -377,16 +340,7 @@ public class ProgramConverter
return tmpPB;
}
-
- /**
- *
- * @param pfpb
- * @param pid
- * @param IDPrefix
- * @param prog
- * @return
- * @throws DMLRuntimeException
- */
+
public static ParForProgramBlock createDeepCopyParForProgramBlock(ParForProgramBlock pfpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
throws DMLRuntimeException
{
@@ -423,9 +377,15 @@ public class ProgramConverter
* This creates a deep copy of a function program block. The central reference to singletons of function program blocks
* poses the need for explicit copies in order to prevent conflicting writes of temporary variables (see ExternalFunctionProgramBlock.
*
- * @param oldName
- * @param pid
- * @throws DMLRuntimeException
+ * @param namespace function namespace
+ * @param oldName ?
+ * @param pid ?
+ * @param IDPrefix ?
+ * @param prog runtime program
+ * @param fnStack ?
+ * @param fnCreated ?
+ * @param plain ?
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static void createDeepCopyFunctionProgramBlock(String namespace, String oldName, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain)
throws DMLRuntimeException
@@ -486,13 +446,7 @@ public class ProgramConverter
prog.addFunctionProgramBlock(namespace, fnameNew, copy);
fnCreated.add(DMLProgram.constructFunctionKey(namespace, fnameNew));
}
-
- /**
- *
- * @param fpb
- * @return
- * @throws DMLRuntimeException
- */
+
public static FunctionProgramBlock createDeepCopyFunctionProgramBlock(FunctionProgramBlock fpb, HashSet<String> fnStack, HashSet<String> fnCreated)
throws DMLRuntimeException
{
@@ -524,10 +478,16 @@ public class ProgramConverter
* IDs with the concrete IDs of this parfor instance. This is a helper method uses for generating
* deep copies of program blocks.
*
- * @param instSet
- * @param pid
- * @return
- * @throws DMLRuntimeException
+ * @param instSet list of instructions
+ * @param pid ?
+ * @param IDPrefix ?
+ * @param prog runtime program
+ * @param fnStack ?
+ * @param fnCreated ?
+ * @param plain ?
+ * @param cpFunctions ?
+ * @return list of instructions
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static ArrayList<Instruction> createDeepCopyInstructionSet(ArrayList<Instruction> instSet, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean cpFunctions)
throws DMLRuntimeException
@@ -548,14 +508,7 @@ public class ProgramConverter
return tmp;
}
-
-
- /**
- *
- * @param pid
- * @return
- * @throws DMLRuntimeException
- */
+
public static Instruction cloneInstruction( Instruction oInst, long pid, boolean plain, boolean cpFunctions )
throws DMLRuntimeException
{
@@ -599,13 +552,7 @@ public class ProgramConverter
return inst;
}
-
- /**
- *
- * @param sb
- * @return
- * @throws DMLRuntimeException
- */
+
public static StatementBlock createStatementBlockCopy( StatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
throws DMLRuntimeException
{
@@ -645,15 +592,7 @@ public class ProgramConverter
return ret;
}
-
- /**
- *
- * @param sb
- * @param pid
- * @param plain
- * @return
- * @throws DMLRuntimeException
- */
+
public static IfStatementBlock createIfStatementBlockCopy( IfStatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
throws DMLRuntimeException
{
@@ -694,15 +633,7 @@ public class ProgramConverter
return ret;
}
-
- /**
- *
- * @param sb
- * @param pid
- * @param plain
- * @return
- * @throws DMLRuntimeException
- */
+
public static WhileStatementBlock createWhileStatementBlockCopy( WhileStatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
throws DMLRuntimeException
{
@@ -744,15 +675,7 @@ public class ProgramConverter
return ret;
}
-
- /**
- *
- * @param sb
- * @param pid
- * @param plain
- * @return
- * @throws DMLRuntimeException
- */
+
public static ForStatementBlock createForStatementBlockCopy( ForStatementBlock sb, long pid, boolean plain, boolean forceDeepCopy )
throws DMLRuntimeException
{
@@ -813,12 +736,7 @@ public class ProgramConverter
////////////////////////////////
// SERIALIZATION
////////////////////////////////
-
- /**
- * @param body
- * @return
- * @throws DMLRuntimeException
- */
+
public static String serializeParForBody( ParForBody body )
throws DMLRuntimeException
{
@@ -884,13 +802,7 @@ public class ProgramConverter
return sb.toString();
}
-
- /**
- *
- * @param prog
- * @return
- * @throws DMLRuntimeException
- */
+
public static String serializeProgram( Program prog, ArrayList<ProgramBlock> pbs )
throws DMLRuntimeException
{
@@ -903,13 +815,7 @@ public class ProgramConverter
return rSerializeFunctionProgramBlocks( fpb, cand );
}
-
- /**
- *
- * @param pbs
- * @param cand
- * @throws DMLRuntimeException
- */
+
public static void rFindSerializationCandidates( ArrayList<ProgramBlock> pbs, HashSet<String> cand )
throws DMLRuntimeException
{
@@ -951,14 +857,7 @@ public class ProgramConverter
}
}
}
-
-
- /**
- *
- * @param vars
- * @return
- * @throws DMLRuntimeException
- */
+
public static String serializeVariables (LocalVariableMap vars)
throws DMLRuntimeException
{
@@ -969,14 +868,7 @@ public class ProgramConverter
return sb.toString();
}
-
- /**
- *
- * @param key
- * @param dat
- * @return
- * @throws DMLRuntimeException
- */
+
public static String serializeDataObject(String key, Data dat)
throws DMLRuntimeException
{
@@ -1037,13 +929,7 @@ public class ProgramConverter
return sb.toString();
}
-
- /**
- *
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
+
public static String serializeExecutionContext( ExecutionContext ec )
throws DMLRuntimeException
{
@@ -1061,13 +947,7 @@ public class ProgramConverter
return ret;
}
-
- /**
- *
- * @param inst
- * @return
- * @throws DMLRuntimeException
- */
+
@SuppressWarnings("all")
public static String serializeInstructions( ArrayList<Instruction> inst )
throws DMLRuntimeException
@@ -1100,8 +980,8 @@ public class ProgramConverter
* (e.g. print( "a,b" ) would break the parsing of instruction that internally
* are separated with a "," )
*
- * @param instStr
- * @return
+ * @param instStr instruction string
+ * @return instruction string with replacements
*/
public static String checkAndReplaceLiterals( String instStr )
{
@@ -1139,13 +1019,7 @@ public class ProgramConverter
return tmp;
}
-
-
- /**
- *
- * @param vars
- * @return
- */
+
public static String serializeStringHashMap( HashMap<String,String> vars)
{
StringBuilder sb = new StringBuilder();
@@ -1161,12 +1035,7 @@ public class ProgramConverter
}
return sb.toString();
}
-
- /**
- *
- * @param vars
- * @return
- */
+
public static String serializeStringCollection( Collection<String> set)
{
StringBuilder sb = new StringBuilder();
@@ -1180,13 +1049,7 @@ public class ProgramConverter
}
return sb.toString();
}
-
-
- /**
- *
- * @param vars
- * @return
- */
+
public static String serializeStringArrayList( ArrayList<String> vars)
{
StringBuilder sb = new StringBuilder();
@@ -1200,12 +1063,7 @@ public class ProgramConverter
}
return sb.toString();
}
-
- /**
- *
- * @param vars
- * @return
- */
+
public static String serializeStringArray( String[] vars)
{
StringBuilder sb = new StringBuilder();
@@ -1224,11 +1082,6 @@ public class ProgramConverter
return sb.toString();
}
- /**
- *
- * @param var
- * @return
- */
public static String serializeDataIdentifiers( ArrayList<DataIdentifier> var)
{
StringBuilder sb = new StringBuilder();
@@ -1242,12 +1095,7 @@ public class ProgramConverter
}
return sb.toString();
}
-
- /**
- *
- * @param dat
- * @return
- */
+
public static String serializeDataIdentifier( DataIdentifier dat )
{
// SCHEMA: <name>|<datatype>|<valuetype>
@@ -1261,13 +1109,7 @@ public class ProgramConverter
return sb.toString();
}
-
- /**
- *
- * @param pbs
- * @return
- * @throws DMLRuntimeException
- */
+
public static String rSerializeFunctionProgramBlocks(HashMap<String,FunctionProgramBlock> pbs, HashSet<String> cand)
throws DMLRuntimeException
{
@@ -1293,13 +1135,7 @@ public class ProgramConverter
sb.append(NEWLINE);
return sb.toString();
}
-
- /**
- *
- * @param pbs
- * @return
- * @throws DMLRuntimeException
- */
+
public static String rSerializeProgramBlocks(ArrayList<ProgramBlock> pbs)
throws DMLRuntimeException
{
@@ -1318,13 +1154,7 @@ public class ProgramConverter
return sb.toString();
}
-
- /**
- *
- * @param pb
- * @return
- * @throws DMLRuntimeException
- */
+
public static String rSerializeProgramBlock( ProgramBlock pb)
throws DMLRuntimeException
{
@@ -1513,15 +1343,7 @@ public class ProgramConverter
////////////////////////////////
// PARSING
////////////////////////////////
-
-
- /**
- *
- * @param in
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static ParForBody parseParForBody( String in, int id )
throws DMLRuntimeException
{
@@ -1575,14 +1397,7 @@ public class ProgramConverter
return body;
}
-
- /**
- *
- * @param in
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static Program parseProgram( String in, int id )
throws DMLRuntimeException
{
@@ -1602,13 +1417,7 @@ public class ProgramConverter
return prog;
}
-
- /**
- *
- * @param in
- * @return
- * @throws DMLRuntimeException
- */
+
public static LocalVariableMap parseVariables(String in)
throws DMLRuntimeException
{
@@ -1626,15 +1435,7 @@ public class ProgramConverter
return ret;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static HashMap<String,FunctionProgramBlock> parseFunctionProgramBlocks( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1660,15 +1461,7 @@ public class ProgramConverter
return ret;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static ArrayList<ProgramBlock> rParseProgramBlocks(String in, Program prog, int id)
throws DMLRuntimeException
{
@@ -1684,15 +1477,7 @@ public class ProgramConverter
return pbs;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static ProgramBlock rParseProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1718,14 +1503,6 @@ public class ProgramConverter
return pb;
}
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
public static WhileProgramBlock rParseWhileProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1752,15 +1529,7 @@ public class ProgramConverter
return wpb;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static ForProgramBlock rParseForProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1793,15 +1562,7 @@ public class ProgramConverter
return fpb;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static ParForProgramBlock rParseParForProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1838,15 +1599,7 @@ public class ProgramConverter
return pfpb;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static IfProgramBlock rParseIfProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1875,15 +1628,7 @@ public class ProgramConverter
return ipb;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static FunctionProgramBlock rParseFunctionProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1911,15 +1656,7 @@ public class ProgramConverter
return fpb;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static ExternalFunctionProgramBlock rParseExternalFunctionProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1953,15 +1690,7 @@ public class ProgramConverter
return efpb;
}
-
- /**
- *
- * @param in
- * @param prog
- * @param id
- * @return
- * @throws DMLRuntimeException
- */
+
public static ProgramBlock rParseGenericProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
@@ -1975,13 +1704,7 @@ public class ProgramConverter
return pb;
}
-
- /**
- *
- * @param in
- * @return
- * @throws DMLRuntimeException
- */
+
public static ArrayList<Instruction> parseInstructions( String in, int id )
throws DMLRuntimeException
{
@@ -2008,12 +1731,7 @@ public class ProgramConverter
return insts;
}
-
- /**
- *
- * @param in
- * @return
- */
+
public static HashMap<String,String> parseStringHashMap( String in )
{
HashMap<String,String> vars = new HashMap<String, String>();
@@ -2029,12 +1747,7 @@ public class ProgramConverter
return vars;
}
-
- /**
- *
- * @param in
- * @return
- */
+
public static ArrayList<String> parseStringArrayList( String in )
{
ArrayList<String> vars = new ArrayList<String>();
@@ -2047,12 +1760,7 @@ public class ProgramConverter
return vars;
}
-
- /**
- *
- * @param in
- * @return
- */
+
public static String[] parseStringArray( String in )
{
StringTokenizer st = new StringTokenizer(in, ELEMENT_DELIM);
@@ -2069,11 +1777,6 @@ public class ProgramConverter
return a;
}
- /**
- *
- * @param in
- * @return
- */
public static ArrayList<DataIdentifier> parseDataIdentifiers( String in )
{
ArrayList<DataIdentifier> vars = new ArrayList<DataIdentifier>();
@@ -2087,12 +1790,7 @@ public class ProgramConverter
return vars;
}
-
- /**
- *
- * @param in
- * @return
- */
+
public static DataIdentifier parseDataIdentifier( String in )
{
StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
@@ -2111,9 +1809,9 @@ public class ProgramConverter
* NOTE: MRJobConfiguration cannot be used for the general case because program blocks and
* related symbol tables can be hierarchically structured.
*
- * @param in
- * @return
- * @throws DMLRuntimeException
+ * @param in data object as string
+ * @return array of objects
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static Object[] parseDataObject(String in)
throws DMLRuntimeException
@@ -2182,13 +1880,7 @@ public class ProgramConverter
ret[1] = dat;
return ret;
}
-
- /**
- *
- * @param in
- * @return
- * @throws DMLRuntimeException
- */
+
public static ExecutionContext parseExecutionContext(String in, Program prog)
throws DMLRuntimeException
{
@@ -2219,11 +1911,11 @@ public class ProgramConverter
/**
* In-place replacement of thread ids in filenames, functions names etc
*
- * @param instInst
- * @param pattern
- * @param replacement
- * @return
- * @throws DMLRuntimeException
+ * @param inst instruction
+ * @param pattern ?
+ * @param replacement string replacement
+ * @return instruction
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static Instruction saveReplaceThreadID( Instruction inst, String pattern, String replacement )
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
index 15b1fb3..39af0e5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
@@ -69,20 +69,7 @@ public class RemoteDPParForMR
{
protected static final Log LOG = LogFactory.getLog(RemoteDPParForMR.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param taskFile
- * @param resultFile
- * @param enableCPCaching
- * @param mode
- * @param numMappers
- * @param replication
- * @return
- * @throws DMLRuntimeException
- */
+
public static RemoteParForJobReturn runJob(long pfid, String itervar, String matrixvar, String program, String resultFile, MatrixObject input,
PDataPartitionFormat dpf, OutputInfo oi, boolean tSparseCol, //config params
boolean enableCPCaching, int numReducers, int replication, int max_retry) //opt params
@@ -247,10 +234,11 @@ public class RemoteDPParForMR
* (the RemoteParWorkerMapper ensures uniqueness of those files independent of the
* runtime implementation).
*
- * @param job
- * @param fname
- * @return
- * @throws DMLRuntimeException
+ * @param job job configuration
+ * @param fname file name
+ * @return array of local variable maps
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
+ * @throws IOException if IOException occurs
*/
@SuppressWarnings("deprecation")
public static LocalVariableMap [] readResultFile( JobConf job, String fname )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
index 484888e..7286eca 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java
@@ -54,20 +54,7 @@ public class RemoteDPParForSpark
{
protected static final Log LOG = LogFactory.getLog(RemoteDPParForSpark.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param taskFile
- * @param resultFile
- * @param enableCPCaching
- * @param mode
- * @param numMappers
- * @param replication
- * @return
- * @throws DMLRuntimeException
- */
+
public static RemoteParForJobReturn runJob(long pfid, String itervar, String matrixvar, String program, String resultFile, MatrixObject input,
ExecutionContext ec,
PDataPartitionFormat dpf, OutputInfo oi, boolean tSparseCol, //config params
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
index 318a71e..2a0dc7d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
@@ -44,9 +44,6 @@ import org.apache.sysml.runtime.util.LocalFileUtils;
import scala.Tuple2;
-/**
- *
- */
public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapFunction<Iterator<Tuple2<Long, Iterable<Writable>>>, Long, String>
{
private static final long serialVersionUID = 30223759283155139L;
@@ -142,13 +139,7 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF
return ret;
}
-
- /**
- *
- * @param ID
- * @throws DMLRuntimeException
- * @throws IOException
- */
+
private void configureWorker( long ID )
throws DMLRuntimeException, IOException
{
@@ -187,9 +178,9 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF
* Note it reuses the instance attribute _partition - multiple calls
* will overwrite the result.
*
- * @param valueList
- * @return
- * @throws IOException
+ * @param valueList iterable writables
+ * @return matrix block
+ * @throws IOException if IOException occurs
*/
private MatrixBlock collectBinaryBlock( Iterable<Writable> valueList )
throws IOException
@@ -241,9 +232,9 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF
* Note it reuses the instance attribute _partition - multiple calls
* will overwrite the result.
*
- * @param valueList
- * @return
- * @throws IOException
+ * @param valueList iterable writables
+ * @return matrix block
+ * @throws IOException if IOException occurs
*/
private MatrixBlock collectBinaryCellInput( Iterable<Writable> valueList )
throws IOException
@@ -288,12 +279,7 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF
return partition;
}
-
- /**
- *
- * @param sort
- * @throws IOException
- */
+
private void cleanupCollectedMatrixPartition(MatrixBlock partition, boolean sort)
throws IOException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
index 893a2ac..ac6d42a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParWorkerReducer.java
@@ -50,9 +50,6 @@ import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
import org.apache.sysml.runtime.util.LocalFileUtils;
import org.apache.sysml.utils.Statistics;
-/**
- *
- */
public class RemoteDPParWorkerReducer extends ParWorker
implements Reducer<LongWritable, Writable, Writable, Writable>
{
@@ -78,10 +75,7 @@ public class RemoteDPParWorkerReducer extends ParWorker
//cached collector/reporter
protected OutputCollector<Writable, Writable> _out = null;
protected Reporter _report = null;
-
- /**
- *
- */
+
public RemoteDPParWorkerReducer()
{
@@ -124,9 +118,6 @@ public class RemoteDPParWorkerReducer extends ParWorker
RemoteParForUtils.incrementParForMRCounters(_report, 1, 1);
}
- /**
- *
- */
@Override
public void configure(JobConf job)
{
@@ -210,10 +201,7 @@ public class RemoteDPParWorkerReducer extends ParWorker
Statistics.reset();
}
}
-
- /**
- *
- */
+
@Override
public void close()
throws IOException
@@ -250,9 +238,9 @@ public class RemoteDPParWorkerReducer extends ParWorker
* Note it reuses the instance attribute _partition - multiple calls
* will overwrite the result.
*
- * @param valueList
- * @return
- * @throws IOException
+ * @param valueList iterable writables
+ * @return matrix block
+ * @throws IOException if IOException occurs
*/
private MatrixBlock collectBinaryBlock( Iterator<Writable> valueList )
throws IOException
@@ -299,9 +287,9 @@ public class RemoteDPParWorkerReducer extends ParWorker
* Note it reuses the instance attribute _partition - multiple calls
* will overwrite the result.
*
- * @param valueList
- * @return
- * @throws IOException
+ * @param valueList iterable writables
+ * @return matrix block
+ * @throws IOException if IOException occurs
*/
private MatrixBlock collectBinaryCellInput( Iterator<Writable> valueList )
throws IOException
@@ -344,12 +332,7 @@ public class RemoteDPParWorkerReducer extends ParWorker
return _partition;
}
-
- /**
- *
- * @param sort
- * @throws IOException
- */
+
private void cleanupCollectedMatrixPartition(boolean sort)
throws IOException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
index 2bbb63a..0b01569 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java
@@ -142,12 +142,6 @@ public class RemoteParForColocatedFileSplit extends FileSplit
return getTopHosts(hosts);
}
-
- /**
- *
- * @param hosts
- * @param names
- */
private void countHosts( HashMap<String,Integer> hosts, String[] names )
{
for( String name : names )
@@ -159,12 +153,7 @@ public class RemoteParForColocatedFileSplit extends FileSplit
hosts.put(name, 1);
}
}
-
- /**
- *
- * @param hosts
- * @return
- */
+
private String[] getTopHosts( HashMap<String,Integer> hosts )
{
int max = Integer.MIN_VALUE;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
index 71bf2da..ef0856e 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForJobReturn.java
@@ -42,46 +42,26 @@ public class RemoteParForJobReturn
_variables = variables;
}
- /**
- *
- * @return
- */
public boolean isSuccessful()
{
return _successful;
}
-
- /**
- *
- * @return
- */
+
public int getNumExecutedTasks()
{
return _numTasks;
}
-
- /**
- *
- * @return
- */
+
public int getNumExecutedIterations()
{
return _numIters;
}
-
- /**
- *
- * @return
- */
+
public LocalVariableMap [] getVariables()
{
return _variables;
}
-
- /**
- *
- * @param variables
- */
+
public void setVariables (LocalVariableMap [] variables)
{
_variables = variables;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
index 7710fe8..7c9332d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
@@ -66,20 +66,7 @@ public class RemoteParForMR
{
protected static final Log LOG = LogFactory.getLog(RemoteParForMR.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param taskFile
- * @param resultFile
- * @param _enableCPCaching
- * @param mode
- * @param numMappers
- * @param replication
- * @return
- * @throws DMLRuntimeException
- */
+
public static RemoteParForJobReturn runJob(long pfid, String program, String taskFile, String resultFile, MatrixObject colocatedDPMatrixObj, //inputs
boolean enableCPCaching, int numMappers, int replication, int max_retry, long minMem, boolean jvmReuse) //opt params
throws DMLRuntimeException
@@ -258,10 +245,11 @@ public class RemoteParForMR
* (the RemoteParWorkerMapper ensures uniqueness of those files independent of the
* runtime implementation).
*
- * @param job
- * @param fname
- * @return
- * @throws DMLRuntimeException
+ * @param job job configuration
+ * @param fname file name
+ * @return array of local variable maps
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
+ * @throws IOException if IOException occurs
*/
@SuppressWarnings("deprecation")
public static LocalVariableMap [] readResultFile( JobConf job, String fname )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
index c228840..47ea2e6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
@@ -52,18 +52,7 @@ public class RemoteParForSpark
{
protected static final Log LOG = LogFactory.getLog(RemoteParForSpark.class.getName());
-
- /**
- *
- * @param pfid
- * @param program
- * @param tasks
- * @param ec
- * @param enableCPCaching
- * @param numMappers
- * @return
- * @throws DMLRuntimeException
- */
+
public static RemoteParForJobReturn runJob(long pfid, String program, List<Task> tasks, ExecutionContext ec,
boolean cpCaching, int numMappers)
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
index 18b5ca2..4dc5126 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java
@@ -33,10 +33,6 @@ import org.apache.sysml.runtime.util.LocalFileUtils;
import scala.Tuple2;
-/**
- *
- *
- */
public class RemoteParForSparkWorker extends ParWorker implements PairFlatMapFunction<Task, Long, String>
{
@@ -88,13 +84,7 @@ public class RemoteParForSparkWorker extends ParWorker implements PairFlatMapFun
return ret;
}
-
- /**
- *
- * @param ID
- * @throws DMLRuntimeException
- * @throws IOException
- */
+
private void configureWorker( long ID )
throws DMLRuntimeException, IOException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
index 67beff6..7b3ecb1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java
@@ -56,13 +56,7 @@ import org.apache.sysml.utils.Statistics;
*/
public class RemoteParForUtils
{
-
- /**
- *
- * @param reporter
- * @param deltaTasks
- * @param deltaIterations
- */
+
public static void incrementParForMRCounters(Reporter reporter, long deltaTasks, long deltaIterations)
{
//report parfor counters
@@ -94,16 +88,7 @@ public class RemoteParForUtils
CacheStatistics.reset();
}
}
-
- /**
- *
- * @param workerID
- * @param vars
- * @param resultVars
- * @param out
- * @throws DMLRuntimeException
- * @throws IOException
- */
+
public static void exportResultVariables( long workerID, LocalVariableMap vars, ArrayList<String> resultVars, OutputCollector<Writable, Writable> out )
throws DMLRuntimeException, IOException
{
@@ -113,13 +98,13 @@ public class RemoteParForUtils
/**
* For remote MR parfor workers.
*
- * @param workerID
- * @param vars
- * @param resultVars
- * @param rvarFnames
- * @param out
- * @throws DMLRuntimeException
- * @throws IOException
+ * @param workerID worker id
+ * @param vars local variable map
+ * @param resultVars list of result variables
+ * @param rvarFnames ?
+ * @param out output collector
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
+ * @throws IOException if IOException occurs
*/
public static void exportResultVariables( long workerID, LocalVariableMap vars, ArrayList<String> resultVars,
HashMap<String,String> rvarFnames, OutputCollector<Writable, Writable> out )
@@ -169,12 +154,12 @@ public class RemoteParForUtils
/**
* For remote Spark parfor workers. This is a simplified version compared to MR.
*
- * @param workerID
- * @param vars
- * @param resultVars
- * @param rvarFnames
- * @throws DMLRuntimeException
- * @throws IOException
+ * @param workerID worker id
+ * @param vars local variable map
+ * @param resultVars list of result variables
+ * @return list of result variables
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
+ * @throws IOException if IOException occurs
*/
public static ArrayList<String> exportResultVariables( long workerID, LocalVariableMap vars, ArrayList<String> resultVars)
throws DMLRuntimeException, IOException
@@ -230,14 +215,7 @@ public class RemoteParForUtils
LocalFileUtils.cleanupWorkingDirectory();
}
}
-
- /**
- *
- * @param out
- * @return
- * @throws DMLRuntimeException
- * @throws IOException
- */
+
public static LocalVariableMap[] getResults( List<Tuple2<Long,String>> out, Log LOG )
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
index ae68b68..4befedc 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java
@@ -77,10 +77,7 @@ public class RemoteParWorkerMapper extends ParWorker //MapReduceBase not requir
//filenames across tasks of one reused worker (preaggregation)
_rvarFnames = new HashMap<String, String>();
}
-
- /**
- *
- */
+
@Override
public void map(LongWritable key, Text value, OutputCollector<Writable, Writable> out, Reporter reporter)
throws IOException
@@ -116,9 +113,6 @@ public class RemoteParWorkerMapper extends ParWorker //MapReduceBase not requir
LOG.info("\nSystemML Statistics:\nHeavy hitter instructions (name, time, count):\n" + Statistics.getHeavyHitters(DMLScript.STATISTICS_COUNT));
}
- /**
- *
- */
@Override
public void configure(JobConf job)
{
@@ -224,9 +218,6 @@ public class RemoteParWorkerMapper extends ParWorker //MapReduceBase not requir
}
}
- /**
- *
- */
@Override
public void close()
throws IOException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
index aa7c1c1..2b10a86 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java
@@ -65,7 +65,7 @@ public abstract class ResultMerge
* of one input matrix at a time.
*
* @return output (merged) matrix
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public abstract MatrixObject executeSerialMerge()
throws DMLRuntimeException;
@@ -77,17 +77,18 @@ public abstract class ResultMerge
*
* @param par degree of parallelism
* @return output (merged) matrix
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public abstract MatrixObject executeParallelMerge( int par )
throws DMLRuntimeException;
/**
+ * ?
*
* @param out initially empty block
- * @param in
- * @param appendOnly
- * @throws DMLRuntimeException
+ * @param in input matrix block
+ * @param appendOnly ?
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
protected void mergeWithoutComp( MatrixBlock out, MatrixBlock in, boolean appendOnly )
throws DMLRuntimeException
@@ -100,9 +101,10 @@ public abstract class ResultMerge
* NOTE: append only not applicable for wiht compare because output must be populated with
* initial state of matrix - with append, this would result in duplicates.
*
- * @param out
- * @param in
- * @throws DMLRuntimeException
+ * @param out output matrix block
+ * @param in input matrix block
+ * @param compare ?
+ * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
protected void mergeWithComp( MatrixBlock out, MatrixBlock in, double[][] compare )
throws DMLRuntimeException
@@ -161,12 +163,7 @@ public abstract class ResultMerge
return ret;
}
-
- /**
- *
- * @param in
- * @return
- */
+
protected ArrayList<MatrixObject> convertToList(MatrixObject[] in)
{
ArrayList<MatrixObject> ret = new ArrayList<MatrixObject>();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
index 44aaf2a..fe190ed 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalAutomatic.java
@@ -27,9 +27,6 @@ import org.apache.sysml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-/**
- *
- */
public class ResultMergeLocalAutomatic extends ResultMerge
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/56e45f1a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
index 9c7e7d3..7e50f0b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
@@ -152,13 +152,6 @@ public class ResultMergeLocalFile extends ResultMerge
return executeSerialMerge();
}
- /**
- *
- * @param output
- * @param inMO
- * @return
- * @throws DMLRuntimeException
- */
private MatrixObject createNewMatrixObject(MatrixObject output, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
@@ -182,14 +175,7 @@ public class ResultMergeLocalFile extends ResultMerge
return moNew;
}
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
+
private void merge( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
@@ -218,14 +204,7 @@ public class ResultMergeLocalFile extends ResultMerge
mergeBinaryBlockWithoutComp( fnameNew, outMo, inMO );
}
}
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
+
private void mergeTextCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
@@ -294,14 +273,7 @@ public class ResultMergeLocalFile extends ResultMerge
throw new DMLRuntimeException("Unable to merge text cell results.", ex);
}
}
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
+
private void mergeTextCellWithComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
@@ -337,14 +309,7 @@ public class ResultMergeLocalFile extends ResultMerge
LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
LocalFileUtils.cleanupWorkingDirectory(fnameStagingCompare);
}
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
+
@SuppressWarnings("deprecation")
private void mergeBinaryCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
@@ -407,14 +372,7 @@ public class ResultMergeLocalFile extends ResultMerge
throw new DMLRuntimeException("Unable to merge binary cell results.", ex);
}
}
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
+
private void mergeBinaryCellWithComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
@@ -450,14 +408,7 @@ public class ResultMergeLocalFile extends ResultMerge
LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
LocalFileUtils.cleanupWorkingDirectory(fnameStagingCompare);
}
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
+
private void mergeBinaryBlockWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
@@ -486,14 +437,7 @@ public class ResultMergeLocalFile extends ResultMerge
LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
}
-
- /**
- *
- * @param fnameNew
- * @param outMo
- * @param inMO
- * @throws DMLRuntimeException
- */
+
private void mergeBinaryBlockWithComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
@@ -528,13 +472,7 @@ public class ResultMergeLocalFile extends ResultMerge
LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
LocalFileUtils.cleanupWorkingDirectory(fnameStagingCompare);
}
-
- /**
- *
- * @param fnameStaging
- * @param mo
- * @throws IOException
- */
+
@SuppressWarnings("deprecation")
private void createBinaryBlockStagingFile( String fnameStaging, MatrixObject mo )
throws IOException
@@ -569,16 +507,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
}
-
- /**
- *
- * @param fnameStaging
- * @param mo
- * @param ID
- * @throws IOException
- * @throws DMLRuntimeException
- */
-
+
private void createTextCellStagingFile( String fnameStaging, MatrixObject mo, long ID )
throws IOException, DMLRuntimeException
{
@@ -641,15 +570,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
}
-
- /**
- *
- * @param fnameStaging
- * @param mo
- * @param ID
- * @throws IOException
- * @throws DMLRuntimeException
- */
+
@SuppressWarnings("deprecation")
private void createBinaryCellStagingFile( String fnameStaging, MatrixObject mo, long ID )
throws IOException, DMLRuntimeException
@@ -697,16 +618,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
}
-
- /**
- * @param fnameStaging
- * @param ID
- * @param buffer
- * @param brlen
- * @param bclen
- * @throws DMLRuntimeException
- * @throws IOException
- */
+
private void appendCellBufferToStagingArea( String fnameStaging, long ID, LinkedList<Cell> buffer, int brlen, int bclen )
throws DMLRuntimeException, IOException
{
@@ -744,17 +656,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
}
-
- /**
- *
- * @param fnameStaging
- * @param fnameStagingCompare
- * @param fnameNew
- * @param metadata
- * @param withCompare
- * @throws IOException
- * @throws DMLRuntimeException
- */
+
@SuppressWarnings("deprecation")
private void createBinaryBlockResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
throws IOException, DMLRuntimeException
@@ -853,17 +755,7 @@ public class ResultMergeLocalFile extends ResultMerge
writer.close();
}
}
-
- /**
- *
- * @param fnameStaging
- * @param fnameStagingCompare
- * @param fnameNew
- * @param metadata
- * @param withCompare
- * @throws IOException
- * @throws DMLRuntimeException
- */
+
private void createTextCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
throws IOException, DMLRuntimeException
{
@@ -1002,16 +894,6 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
- /**
- *
- * @param fnameStaging
- * @param fnameStagingCompare
- * @param fnameNew
- * @param metadata
- * @param withCompare
- * @throws IOException
- * @throws DMLRuntimeException
- */
@SuppressWarnings("deprecation")
private void createBinaryCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
throws IOException, DMLRuntimeException
@@ -1142,13 +1024,6 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
- /**
- *
- * @param fnameNew
- * @param inMO
- * @throws CacheException
- * @throws IOException
- */
private void copyAllFiles( String fnameNew, ArrayList<MatrixObject> inMO )
throws CacheException, IOException
{